diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
65 files changed, 20032 insertions, 7524 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 73ed59eff13..437e1824d0b 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -5,6 +5,7 @@ config DRM_I915 depends on (AGP || AGP=n) select INTEL_GTT select AGP_INTEL if AGP + select INTERVAL_TREE # we need shmfs for the swappable backing store, and in particular # the shmem_readpage() which depends upon tmpfs select SHMEM @@ -14,7 +15,6 @@ config DRM_I915 # but for select to work, need to select ACPI_VIDEO's dependencies, ick select BACKLIGHT_LCD_SUPPORT if ACPI select BACKLIGHT_CLASS_DEVICE if ACPI - select VIDEO_OUTPUT_CONTROL if ACPI select INPUT if ACPI select ACPI_VIDEO if ACPI select ACPI_BUTTON if ACPI @@ -72,7 +72,7 @@ config DRM_I915_PRELIMINARY_HW_SUPPORT config DRM_I915_UMS bool "Enable userspace modesetting on Intel hardware (DEPRECATED)" - depends on DRM_I915 + depends on DRM_I915 && BROKEN default n help Choose this option if you still need userspace modesetting. diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 9fd44f5f3b3..cad1683d8bb 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -3,57 +3,77 @@ # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. ccflags-y := -Iinclude/drm -i915-y := i915_drv.o i915_dma.o i915_irq.o \ - i915_gpu_error.o \ + +# Please keep these build lists sorted! + +# core driver code +i915-y := i915_drv.o \ + i915_params.o \ i915_suspend.o \ - i915_gem.o \ + i915_sysfs.o \ + intel_pm.o +i915-$(CONFIG_COMPAT) += i915_ioc32.o +i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o + +# GEM code +i915-y += i915_cmd_parser.o \ i915_gem_context.o \ + i915_gem_render_state.o \ i915_gem_debug.o \ + i915_gem_dmabuf.o \ i915_gem_evict.o \ i915_gem_execbuffer.o \ i915_gem_gtt.o \ + i915_gem.o \ i915_gem_stolen.o \ i915_gem_tiling.o \ - i915_sysfs.o \ + i915_gem_userptr.o \ + i915_gpu_error.o \ + i915_irq.o \ i915_trace_points.o \ - i915_ums.o \ + intel_ringbuffer.o \ + intel_uncore.o + +# autogenerated null render state +i915-y += intel_renderstate_gen6.o \ + intel_renderstate_gen7.o \ + intel_renderstate_gen8.o + +# modesetting core code +i915-y += intel_bios.o \ intel_display.o \ - intel_crt.o \ - intel_lvds.o \ - intel_dsi.o \ - intel_dsi_cmd.o \ - intel_dsi_pll.o \ - intel_bios.o \ - intel_ddi.o \ - intel_dp.o \ - intel_hdmi.o \ - intel_sdvo.o \ intel_modes.o \ - intel_panel.o \ - intel_pm.o \ - intel_i2c.o \ - intel_tv.o \ - intel_dvo.o \ - intel_ringbuffer.o \ intel_overlay.o \ - intel_sprite.o \ intel_sideband.o \ - intel_uncore.o \ + intel_sprite.o +i915-$(CONFIG_ACPI) += intel_acpi.o intel_opregion.o +i915-$(CONFIG_DRM_I915_FBDEV) += intel_fbdev.o + +# modesetting output/encoder code +i915-y += dvo_ch7017.o \ dvo_ch7xxx.o \ - dvo_ch7017.o \ dvo_ivch.o \ - dvo_tfp410.o \ - dvo_sil164.o \ dvo_ns2501.o \ - i915_gem_dmabuf.o - -i915-$(CONFIG_COMPAT) += i915_ioc32.o - -i915-$(CONFIG_ACPI) += intel_acpi.o intel_opregion.o - -i915-$(CONFIG_DRM_I915_FBDEV) += intel_fbdev.o + dvo_sil164.o \ + dvo_tfp410.o \ + intel_crt.o \ + intel_ddi.o \ + intel_dp.o \ + intel_dsi_cmd.o \ + intel_dsi.o \ + intel_dsi_pll.o \ + intel_dsi_panel_vbt.o \ + intel_dvo.o \ + intel_hdmi.o \ + intel_i2c.o \ + intel_lvds.o \ + intel_panel.o \ + intel_sdvo.o \ + intel_tv.o -i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o +# legacy horrors +i915-y += i915_dma.o \ + i915_ums.o obj-$(CONFIG_DRM_I915) += i915.o diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c index af42e94f684..80449f47596 100644 --- a/drivers/gpu/drm/i915/dvo_ch7xxx.c +++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c @@ -160,7 +160,7 @@ static bool ch7xxx_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch) if (i2c_transfer(adapter, msgs, 2) == 2) { *ch = in_buf[0]; return true; - }; + } if (!ch7xxx->quiet) { DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n", @@ -340,9 +340,9 @@ static void ch7xxx_dump_regs(struct intel_dvo_device *dvo) for (i = 0; i < CH7xxx_NUM_REGS; i++) { uint8_t val; if ((i % 8) == 0) - DRM_LOG_KMS("\n %02X: ", i); + DRM_DEBUG_KMS("\n %02X: ", i); ch7xxx_readb(dvo, i, &val); - DRM_LOG_KMS("%02X ", val); + DRM_DEBUG_KMS("%02X ", val); } } diff --git a/drivers/gpu/drm/i915/dvo_ivch.c b/drivers/gpu/drm/i915/dvo_ivch.c index baaf65bf0bd..0f2587ff347 100644 --- a/drivers/gpu/drm/i915/dvo_ivch.c +++ b/drivers/gpu/drm/i915/dvo_ivch.c @@ -195,7 +195,7 @@ static bool ivch_read(struct intel_dvo_device *dvo, int addr, uint16_t *data) if (i2c_transfer(adapter, msgs, 3) == 3) { *data = (in_buf[1] << 8) | in_buf[0]; return true; - }; + } if (!priv->quiet) { DRM_DEBUG_KMS("Unable to read register 0x%02x from " @@ -377,41 +377,41 @@ static void ivch_dump_regs(struct intel_dvo_device *dvo) uint16_t val; ivch_read(dvo, VR00, &val); - DRM_LOG_KMS("VR00: 0x%04x\n", val); + DRM_DEBUG_KMS("VR00: 0x%04x\n", val); ivch_read(dvo, VR01, &val); - DRM_LOG_KMS("VR01: 0x%04x\n", val); + DRM_DEBUG_KMS("VR01: 0x%04x\n", val); ivch_read(dvo, VR30, &val); - DRM_LOG_KMS("VR30: 0x%04x\n", val); + DRM_DEBUG_KMS("VR30: 0x%04x\n", val); ivch_read(dvo, VR40, &val); - DRM_LOG_KMS("VR40: 0x%04x\n", val); + DRM_DEBUG_KMS("VR40: 0x%04x\n", val); /* GPIO registers */ ivch_read(dvo, VR80, &val); - DRM_LOG_KMS("VR80: 0x%04x\n", val); + DRM_DEBUG_KMS("VR80: 0x%04x\n", val); ivch_read(dvo, VR81, &val); - DRM_LOG_KMS("VR81: 0x%04x\n", val); + DRM_DEBUG_KMS("VR81: 0x%04x\n", val); ivch_read(dvo, VR82, &val); - DRM_LOG_KMS("VR82: 0x%04x\n", val); + DRM_DEBUG_KMS("VR82: 0x%04x\n", val); ivch_read(dvo, VR83, &val); - DRM_LOG_KMS("VR83: 0x%04x\n", val); + DRM_DEBUG_KMS("VR83: 0x%04x\n", val); ivch_read(dvo, VR84, &val); - DRM_LOG_KMS("VR84: 0x%04x\n", val); + DRM_DEBUG_KMS("VR84: 0x%04x\n", val); ivch_read(dvo, VR85, &val); - DRM_LOG_KMS("VR85: 0x%04x\n", val); + DRM_DEBUG_KMS("VR85: 0x%04x\n", val); ivch_read(dvo, VR86, &val); - DRM_LOG_KMS("VR86: 0x%04x\n", val); + DRM_DEBUG_KMS("VR86: 0x%04x\n", val); ivch_read(dvo, VR87, &val); - DRM_LOG_KMS("VR87: 0x%04x\n", val); + DRM_DEBUG_KMS("VR87: 0x%04x\n", val); ivch_read(dvo, VR88, &val); - DRM_LOG_KMS("VR88: 0x%04x\n", val); + DRM_DEBUG_KMS("VR88: 0x%04x\n", val); /* Scratch register 0 - AIM Panel type */ ivch_read(dvo, VR8E, &val); - DRM_LOG_KMS("VR8E: 0x%04x\n", val); + DRM_DEBUG_KMS("VR8E: 0x%04x\n", val); /* Scratch register 1 - Status register */ ivch_read(dvo, VR8F, &val); - DRM_LOG_KMS("VR8F: 0x%04x\n", val); + DRM_DEBUG_KMS("VR8F: 0x%04x\n", val); } static void ivch_destroy(struct intel_dvo_device *dvo) diff --git a/drivers/gpu/drm/i915/dvo_ns2501.c b/drivers/gpu/drm/i915/dvo_ns2501.c index 954acb2c702..74f2af7c2d3 100644 --- a/drivers/gpu/drm/i915/dvo_ns2501.c +++ b/drivers/gpu/drm/i915/dvo_ns2501.c @@ -121,7 +121,7 @@ static bool ns2501_readb(struct intel_dvo_device *dvo, int addr, uint8_t * ch) if (i2c_transfer(adapter, msgs, 2) == 2) { *ch = in_buf[0]; return true; - }; + } if (!ns->quiet) { DRM_DEBUG_KMS @@ -233,9 +233,8 @@ static enum drm_mode_status ns2501_mode_valid(struct intel_dvo_device *dvo, struct drm_display_mode *mode) { DRM_DEBUG_KMS - ("%s: is mode valid (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d)\n", - __FUNCTION__, mode->hdisplay, mode->htotal, mode->vdisplay, - mode->vtotal); + ("is mode valid (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d)\n", + mode->hdisplay, mode->htotal, mode->vdisplay, mode->vtotal); /* * Currently, these are all the modes I have data from. @@ -261,9 +260,8 @@ static void ns2501_mode_set(struct intel_dvo_device *dvo, struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv); DRM_DEBUG_KMS - ("%s: set mode (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d).\n", - __FUNCTION__, mode->hdisplay, mode->htotal, mode->vdisplay, - mode->vtotal); + ("set mode (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d).\n", + mode->hdisplay, mode->htotal, mode->vdisplay, mode->vtotal); /* * Where do I find the native resolution for which scaling is not required??? @@ -277,8 +275,7 @@ static void ns2501_mode_set(struct intel_dvo_device *dvo, if (mode->hdisplay == 800 && mode->vdisplay == 600) { /* mode 277 */ ns->reg_8_shadow &= ~NS2501_8_BPAS; - DRM_DEBUG_KMS("%s: switching to 800x600\n", - __FUNCTION__); + DRM_DEBUG_KMS("switching to 800x600\n"); /* * No, I do not know where this data comes from. @@ -341,8 +338,7 @@ static void ns2501_mode_set(struct intel_dvo_device *dvo, } else if (mode->hdisplay == 640 && mode->vdisplay == 480) { /* mode 274 */ - DRM_DEBUG_KMS("%s: switching to 640x480\n", - __FUNCTION__); + DRM_DEBUG_KMS("switching to 640x480\n"); /* * No, I do not know where this data comes from. * It is just what the video bios left in the DVO, so @@ -406,8 +402,7 @@ static void ns2501_mode_set(struct intel_dvo_device *dvo, } else if (mode->hdisplay == 1024 && mode->vdisplay == 768) { /* mode 280 */ - DRM_DEBUG_KMS("%s: switching to 1024x768\n", - __FUNCTION__); + DRM_DEBUG_KMS("switching to 1024x768\n"); /* * This might or might not work, actually. I'm silently * assuming here that the native panel resolution is @@ -458,8 +453,7 @@ static void ns2501_dpms(struct intel_dvo_device *dvo, bool enable) struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv); unsigned char ch; - DRM_DEBUG_KMS("%s: Trying set the dpms of the DVO to %i\n", - __FUNCTION__, enable); + DRM_DEBUG_KMS("Trying set the dpms of the DVO to %i\n", enable); ch = ns->reg_8_shadow; @@ -490,15 +484,15 @@ static void ns2501_dump_regs(struct intel_dvo_device *dvo) uint8_t val; ns2501_readb(dvo, NS2501_FREQ_LO, &val); - DRM_LOG_KMS("NS2501_FREQ_LO: 0x%02x\n", val); + DRM_DEBUG_KMS("NS2501_FREQ_LO: 0x%02x\n", val); ns2501_readb(dvo, NS2501_FREQ_HI, &val); - DRM_LOG_KMS("NS2501_FREQ_HI: 0x%02x\n", val); + DRM_DEBUG_KMS("NS2501_FREQ_HI: 0x%02x\n", val); ns2501_readb(dvo, NS2501_REG8, &val); - DRM_LOG_KMS("NS2501_REG8: 0x%02x\n", val); + DRM_DEBUG_KMS("NS2501_REG8: 0x%02x\n", val); ns2501_readb(dvo, NS2501_REG9, &val); - DRM_LOG_KMS("NS2501_REG9: 0x%02x\n", val); + DRM_DEBUG_KMS("NS2501_REG9: 0x%02x\n", val); ns2501_readb(dvo, NS2501_REGC, &val); - DRM_LOG_KMS("NS2501_REGC: 0x%02x\n", val); + DRM_DEBUG_KMS("NS2501_REGC: 0x%02x\n", val); } static void ns2501_destroy(struct intel_dvo_device *dvo) diff --git a/drivers/gpu/drm/i915/dvo_sil164.c b/drivers/gpu/drm/i915/dvo_sil164.c index 4debd32e3e4..fa011496707 100644 --- a/drivers/gpu/drm/i915/dvo_sil164.c +++ b/drivers/gpu/drm/i915/dvo_sil164.c @@ -93,7 +93,7 @@ static bool sil164_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch) if (i2c_transfer(adapter, msgs, 2) == 2) { *ch = in_buf[0]; return true; - }; + } if (!sil->quiet) { DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n", @@ -246,15 +246,15 @@ static void sil164_dump_regs(struct intel_dvo_device *dvo) uint8_t val; sil164_readb(dvo, SIL164_FREQ_LO, &val); - DRM_LOG_KMS("SIL164_FREQ_LO: 0x%02x\n", val); + DRM_DEBUG_KMS("SIL164_FREQ_LO: 0x%02x\n", val); sil164_readb(dvo, SIL164_FREQ_HI, &val); - DRM_LOG_KMS("SIL164_FREQ_HI: 0x%02x\n", val); + DRM_DEBUG_KMS("SIL164_FREQ_HI: 0x%02x\n", val); sil164_readb(dvo, SIL164_REG8, &val); - DRM_LOG_KMS("SIL164_REG8: 0x%02x\n", val); + DRM_DEBUG_KMS("SIL164_REG8: 0x%02x\n", val); sil164_readb(dvo, SIL164_REG9, &val); - DRM_LOG_KMS("SIL164_REG9: 0x%02x\n", val); + DRM_DEBUG_KMS("SIL164_REG9: 0x%02x\n", val); sil164_readb(dvo, SIL164_REGC, &val); - DRM_LOG_KMS("SIL164_REGC: 0x%02x\n", val); + DRM_DEBUG_KMS("SIL164_REGC: 0x%02x\n", val); } static void sil164_destroy(struct intel_dvo_device *dvo) diff --git a/drivers/gpu/drm/i915/dvo_tfp410.c b/drivers/gpu/drm/i915/dvo_tfp410.c index e17f1b07e91..7853719a0e8 100644 --- a/drivers/gpu/drm/i915/dvo_tfp410.c +++ b/drivers/gpu/drm/i915/dvo_tfp410.c @@ -118,7 +118,7 @@ static bool tfp410_readb(struct intel_dvo_device *dvo, int addr, uint8_t *ch) if (i2c_transfer(adapter, msgs, 2) == 2) { *ch = in_buf[0]; return true; - }; + } if (!tfp->quiet) { DRM_DEBUG_KMS("Unable to read register 0x%02x from %s:%02x.\n", @@ -267,33 +267,33 @@ static void tfp410_dump_regs(struct intel_dvo_device *dvo) uint8_t val, val2; tfp410_readb(dvo, TFP410_REV, &val); - DRM_LOG_KMS("TFP410_REV: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_REV: 0x%02X\n", val); tfp410_readb(dvo, TFP410_CTL_1, &val); - DRM_LOG_KMS("TFP410_CTL1: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_CTL1: 0x%02X\n", val); tfp410_readb(dvo, TFP410_CTL_2, &val); - DRM_LOG_KMS("TFP410_CTL2: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_CTL2: 0x%02X\n", val); tfp410_readb(dvo, TFP410_CTL_3, &val); - DRM_LOG_KMS("TFP410_CTL3: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_CTL3: 0x%02X\n", val); tfp410_readb(dvo, TFP410_USERCFG, &val); - DRM_LOG_KMS("TFP410_USERCFG: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_USERCFG: 0x%02X\n", val); tfp410_readb(dvo, TFP410_DE_DLY, &val); - DRM_LOG_KMS("TFP410_DE_DLY: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_DE_DLY: 0x%02X\n", val); tfp410_readb(dvo, TFP410_DE_CTL, &val); - DRM_LOG_KMS("TFP410_DE_CTL: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_DE_CTL: 0x%02X\n", val); tfp410_readb(dvo, TFP410_DE_TOP, &val); - DRM_LOG_KMS("TFP410_DE_TOP: 0x%02X\n", val); + DRM_DEBUG_KMS("TFP410_DE_TOP: 0x%02X\n", val); tfp410_readb(dvo, TFP410_DE_CNT_LO, &val); tfp410_readb(dvo, TFP410_DE_CNT_HI, &val2); - DRM_LOG_KMS("TFP410_DE_CNT: 0x%02X%02X\n", val2, val); + DRM_DEBUG_KMS("TFP410_DE_CNT: 0x%02X%02X\n", val2, val); tfp410_readb(dvo, TFP410_DE_LIN_LO, &val); tfp410_readb(dvo, TFP410_DE_LIN_HI, &val2); - DRM_LOG_KMS("TFP410_DE_LIN: 0x%02X%02X\n", val2, val); + DRM_DEBUG_KMS("TFP410_DE_LIN: 0x%02X%02X\n", val2, val); tfp410_readb(dvo, TFP410_H_RES_LO, &val); tfp410_readb(dvo, TFP410_H_RES_HI, &val2); - DRM_LOG_KMS("TFP410_H_RES: 0x%02X%02X\n", val2, val); + DRM_DEBUG_KMS("TFP410_H_RES: 0x%02X%02X\n", val2, val); tfp410_readb(dvo, TFP410_V_RES_LO, &val); tfp410_readb(dvo, TFP410_V_RES_HI, &val2); - DRM_LOG_KMS("TFP410_V_RES: 0x%02X%02X\n", val2, val); + DRM_DEBUG_KMS("TFP410_V_RES: 0x%02X%02X\n", val2, val); } static void tfp410_destroy(struct intel_dvo_device *dvo) diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c new file mode 100644 index 00000000000..9d7954366bd --- /dev/null +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -0,0 +1,1061 @@ +/* + * Copyright © 2013 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Brad Volkin <bradley.d.volkin@intel.com> + * + */ + +#include "i915_drv.h" + +/** + * DOC: batch buffer command parser + * + * Motivation: + * Certain OpenGL features (e.g. transform feedback, performance monitoring) + * require userspace code to submit batches containing commands such as + * MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some + * generations of the hardware will noop these commands in "unsecure" batches + * (which includes all userspace batches submitted via i915) even though the + * commands may be safe and represent the intended programming model of the + * device. + * + * The software command parser is similar in operation to the command parsing + * done in hardware for unsecure batches. However, the software parser allows + * some operations that would be noop'd by hardware, if the parser determines + * the operation is safe, and submits the batch as "secure" to prevent hardware + * parsing. + * + * Threats: + * At a high level, the hardware (and software) checks attempt to prevent + * granting userspace undue privileges. There are three categories of privilege. + * + * First, commands which are explicitly defined as privileged or which should + * only be used by the kernel driver. The parser generally rejects such + * commands, though it may allow some from the drm master process. + * + * Second, commands which access registers. To support correct/enhanced + * userspace functionality, particularly certain OpenGL extensions, the parser + * provides a whitelist of registers which userspace may safely access (for both + * normal and drm master processes). + * + * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc). + * The parser always rejects such commands. + * + * The majority of the problematic commands fall in the MI_* range, with only a + * few specific commands on each ring (e.g. PIPE_CONTROL and MI_FLUSH_DW). + * + * Implementation: + * Each ring maintains tables of commands and registers which the parser uses in + * scanning batch buffers submitted to that ring. + * + * Since the set of commands that the parser must check for is significantly + * smaller than the number of commands supported, the parser tables contain only + * those commands required by the parser. This generally works because command + * opcode ranges have standard command length encodings. So for commands that + * the parser does not need to check, it can easily skip them. This is + * implementated via a per-ring length decoding vfunc. + * + * Unfortunately, there are a number of commands that do not follow the standard + * length encoding for their opcode range, primarily amongst the MI_* commands. + * To handle this, the parser provides a way to define explicit "skip" entries + * in the per-ring command tables. + * + * Other command table entries map fairly directly to high level categories + * mentioned above: rejected, master-only, register whitelist. The parser + * implements a number of checks, including the privileged memory checks, via a + * general bitmasking mechanism. + */ + +#define STD_MI_OPCODE_MASK 0xFF800000 +#define STD_3D_OPCODE_MASK 0xFFFF0000 +#define STD_2D_OPCODE_MASK 0xFFC00000 +#define STD_MFX_OPCODE_MASK 0xFFFF0000 + +#define CMD(op, opm, f, lm, fl, ...) \ + { \ + .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \ + .cmd = { (op), (opm) }, \ + .length = { (lm) }, \ + __VA_ARGS__ \ + } + +/* Convenience macros to compress the tables */ +#define SMI STD_MI_OPCODE_MASK +#define S3D STD_3D_OPCODE_MASK +#define S2D STD_2D_OPCODE_MASK +#define SMFX STD_MFX_OPCODE_MASK +#define F true +#define S CMD_DESC_SKIP +#define R CMD_DESC_REJECT +#define W CMD_DESC_REGISTER +#define B CMD_DESC_BITMASK +#define M CMD_DESC_MASTER + +/* Command Mask Fixed Len Action + ---------------------------------------------------------- */ +static const struct drm_i915_cmd_descriptor common_cmds[] = { + CMD( MI_NOOP, SMI, F, 1, S ), + CMD( MI_USER_INTERRUPT, SMI, F, 1, R ), + CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, M ), + CMD( MI_ARB_CHECK, SMI, F, 1, S ), + CMD( MI_REPORT_HEAD, SMI, F, 1, S ), + CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ), + CMD( MI_SEMAPHORE_MBOX, SMI, !F, 0xFF, R ), + CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ), + CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W, + .reg = { .offset = 1, .mask = 0x007FFFFC } ), + CMD( MI_STORE_REGISTER_MEM(1), SMI, !F, 0xFF, W | B, + .reg = { .offset = 1, .mask = 0x007FFFFC }, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_LOAD_REGISTER_MEM, SMI, !F, 0xFF, W | B, + .reg = { .offset = 1, .mask = 0x007FFFFC }, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ), +}; + +static const struct drm_i915_cmd_descriptor render_cmds[] = { + CMD( MI_FLUSH, SMI, F, 1, S ), + CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), + CMD( MI_PREDICATE, SMI, F, 1, S ), + CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ), + CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), + CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ), + CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ), + CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_UPDATE_GTT, SMI, !F, 0xFF, R ), + CMD( MI_CLFLUSH, SMI, !F, 0x3FF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_REPORT_PERF_COUNT, SMI, !F, 0x3F, B, + .bits = {{ + .offset = 1, + .mask = MI_REPORT_PERF_COUNT_GGTT, + .expected = 0, + }}, ), + CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( GFX_OP_3DSTATE_VF_STATISTICS, S3D, F, 1, S ), + CMD( PIPELINE_SELECT, S3D, F, 1, S ), + CMD( MEDIA_VFE_STATE, S3D, !F, 0xFFFF, B, + .bits = {{ + .offset = 2, + .mask = MEDIA_VFE_STATE_MMIO_ACCESS_MASK, + .expected = 0, + }}, ), + CMD( GPGPU_OBJECT, S3D, !F, 0xFF, S ), + CMD( GPGPU_WALKER, S3D, !F, 0xFF, S ), + CMD( GFX_OP_3DSTATE_SO_DECL_LIST, S3D, !F, 0x1FF, S ), + CMD( GFX_OP_PIPE_CONTROL(5), S3D, !F, 0xFF, B, + .bits = {{ + .offset = 1, + .mask = (PIPE_CONTROL_MMIO_WRITE | PIPE_CONTROL_NOTIFY), + .expected = 0, + }, + { + .offset = 1, + .mask = (PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_STORE_DATA_INDEX), + .expected = 0, + .condition_offset = 1, + .condition_mask = PIPE_CONTROL_POST_SYNC_OP_MASK, + }}, ), +}; + +static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = { + CMD( MI_SET_PREDICATE, SMI, F, 1, S ), + CMD( MI_RS_CONTROL, SMI, F, 1, S ), + CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ), + CMD( MI_RS_CONTEXT, SMI, F, 1, S ), + CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), + CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), + CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, R ), + CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ), + CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ), + CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ), + CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_VS, S3D, !F, 0x7FF, S ), + CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_PS, S3D, !F, 0x7FF, S ), + + CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS, S3D, !F, 0x1FF, S ), + CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS, S3D, !F, 0x1FF, S ), + CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS, S3D, !F, 0x1FF, S ), + CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS, S3D, !F, 0x1FF, S ), + CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ), +}; + +static const struct drm_i915_cmd_descriptor video_cmds[] = { + CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), + CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), + CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, + .bits = {{ + .offset = 0, + .mask = MI_FLUSH_DW_NOTIFY, + .expected = 0, + }, + { + .offset = 1, + .mask = MI_FLUSH_DW_USE_GTT, + .expected = 0, + .condition_offset = 0, + .condition_mask = MI_FLUSH_DW_OP_MASK, + }, + { + .offset = 0, + .mask = MI_FLUSH_DW_STORE_INDEX, + .expected = 0, + .condition_offset = 0, + .condition_mask = MI_FLUSH_DW_OP_MASK, + }}, ), + CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + /* + * MFX_WAIT doesn't fit the way we handle length for most commands. + * It has a length field but it uses a non-standard length bias. + * It is always 1 dword though, so just treat it as fixed length. + */ + CMD( MFX_WAIT, SMFX, F, 1, S ), +}; + +static const struct drm_i915_cmd_descriptor vecs_cmds[] = { + CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), + CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), + CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, + .bits = {{ + .offset = 0, + .mask = MI_FLUSH_DW_NOTIFY, + .expected = 0, + }, + { + .offset = 1, + .mask = MI_FLUSH_DW_USE_GTT, + .expected = 0, + .condition_offset = 0, + .condition_mask = MI_FLUSH_DW_OP_MASK, + }, + { + .offset = 0, + .mask = MI_FLUSH_DW_STORE_INDEX, + .expected = 0, + .condition_offset = 0, + .condition_mask = MI_FLUSH_DW_OP_MASK, + }}, ), + CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), +}; + +static const struct drm_i915_cmd_descriptor blt_cmds[] = { + CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), + CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B, + .bits = {{ + .offset = 0, + .mask = MI_GLOBAL_GTT, + .expected = 0, + }}, ), + CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), + CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, + .bits = {{ + .offset = 0, + .mask = MI_FLUSH_DW_NOTIFY, + .expected = 0, + }, + { + .offset = 1, + .mask = MI_FLUSH_DW_USE_GTT, + .expected = 0, + .condition_offset = 0, + .condition_mask = MI_FLUSH_DW_OP_MASK, + }, + { + .offset = 0, + .mask = MI_FLUSH_DW_STORE_INDEX, + .expected = 0, + .condition_offset = 0, + .condition_mask = MI_FLUSH_DW_OP_MASK, + }}, ), + CMD( COLOR_BLT, S2D, !F, 0x3F, S ), + CMD( SRC_COPY_BLT, S2D, !F, 0x3F, S ), +}; + +static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = { + CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), + CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), +}; + +#undef CMD +#undef SMI +#undef S3D +#undef S2D +#undef SMFX +#undef F +#undef S +#undef R +#undef W +#undef B +#undef M + +static const struct drm_i915_cmd_table gen7_render_cmds[] = { + { common_cmds, ARRAY_SIZE(common_cmds) }, + { render_cmds, ARRAY_SIZE(render_cmds) }, +}; + +static const struct drm_i915_cmd_table hsw_render_ring_cmds[] = { + { common_cmds, ARRAY_SIZE(common_cmds) }, + { render_cmds, ARRAY_SIZE(render_cmds) }, + { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) }, +}; + +static const struct drm_i915_cmd_table gen7_video_cmds[] = { + { common_cmds, ARRAY_SIZE(common_cmds) }, + { video_cmds, ARRAY_SIZE(video_cmds) }, +}; + +static const struct drm_i915_cmd_table hsw_vebox_cmds[] = { + { common_cmds, ARRAY_SIZE(common_cmds) }, + { vecs_cmds, ARRAY_SIZE(vecs_cmds) }, +}; + +static const struct drm_i915_cmd_table gen7_blt_cmds[] = { + { common_cmds, ARRAY_SIZE(common_cmds) }, + { blt_cmds, ARRAY_SIZE(blt_cmds) }, +}; + +static const struct drm_i915_cmd_table hsw_blt_ring_cmds[] = { + { common_cmds, ARRAY_SIZE(common_cmds) }, + { blt_cmds, ARRAY_SIZE(blt_cmds) }, + { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) }, +}; + +/* + * Register whitelists, sorted by increasing register offset. + * + * Some registers that userspace accesses are 64 bits. The register + * access commands only allow 32-bit accesses. Hence, we have to include + * entries for both halves of the 64-bit registers. + */ + +/* Convenience macro for adding 64-bit registers */ +#define REG64(addr) (addr), (addr + sizeof(u32)) + +static const u32 gen7_render_regs[] = { + REG64(HS_INVOCATION_COUNT), + REG64(DS_INVOCATION_COUNT), + REG64(IA_VERTICES_COUNT), + REG64(IA_PRIMITIVES_COUNT), + REG64(VS_INVOCATION_COUNT), + REG64(GS_INVOCATION_COUNT), + REG64(GS_PRIMITIVES_COUNT), + REG64(CL_INVOCATION_COUNT), + REG64(CL_PRIMITIVES_COUNT), + REG64(PS_INVOCATION_COUNT), + REG64(PS_DEPTH_COUNT), + OACONTROL, /* Only allowed for LRI and SRM. See below. */ + GEN7_3DPRIM_END_OFFSET, + GEN7_3DPRIM_START_VERTEX, + GEN7_3DPRIM_VERTEX_COUNT, + GEN7_3DPRIM_INSTANCE_COUNT, + GEN7_3DPRIM_START_INSTANCE, + GEN7_3DPRIM_BASE_VERTEX, + REG64(GEN7_SO_NUM_PRIMS_WRITTEN(0)), + REG64(GEN7_SO_NUM_PRIMS_WRITTEN(1)), + REG64(GEN7_SO_NUM_PRIMS_WRITTEN(2)), + REG64(GEN7_SO_NUM_PRIMS_WRITTEN(3)), + REG64(GEN7_SO_PRIM_STORAGE_NEEDED(0)), + REG64(GEN7_SO_PRIM_STORAGE_NEEDED(1)), + REG64(GEN7_SO_PRIM_STORAGE_NEEDED(2)), + REG64(GEN7_SO_PRIM_STORAGE_NEEDED(3)), + GEN7_SO_WRITE_OFFSET(0), + GEN7_SO_WRITE_OFFSET(1), + GEN7_SO_WRITE_OFFSET(2), + GEN7_SO_WRITE_OFFSET(3), +}; + +static const u32 gen7_blt_regs[] = { + BCS_SWCTRL, +}; + +static const u32 ivb_master_regs[] = { + FORCEWAKE_MT, + DERRMR, + GEN7_PIPE_DE_LOAD_SL(PIPE_A), + GEN7_PIPE_DE_LOAD_SL(PIPE_B), + GEN7_PIPE_DE_LOAD_SL(PIPE_C), +}; + +static const u32 hsw_master_regs[] = { + FORCEWAKE_MT, + DERRMR, +}; + +#undef REG64 + +static u32 gen7_render_get_cmd_length_mask(u32 cmd_header) +{ + u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT; + u32 subclient = + (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; + + if (client == INSTR_MI_CLIENT) + return 0x3F; + else if (client == INSTR_RC_CLIENT) { + if (subclient == INSTR_MEDIA_SUBCLIENT) + return 0xFFFF; + else + return 0xFF; + } + + DRM_DEBUG_DRIVER("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header); + return 0; +} + +static u32 gen7_bsd_get_cmd_length_mask(u32 cmd_header) +{ + u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT; + u32 subclient = + (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; + + if (client == INSTR_MI_CLIENT) + return 0x3F; + else if (client == INSTR_RC_CLIENT) { + if (subclient == INSTR_MEDIA_SUBCLIENT) + return 0xFFF; + else + return 0xFF; + } + + DRM_DEBUG_DRIVER("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header); + return 0; +} + +static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header) +{ + u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT; + + if (client == INSTR_MI_CLIENT) + return 0x3F; + else if (client == INSTR_BC_CLIENT) + return 0xFF; + + DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header); + return 0; +} + +static bool validate_cmds_sorted(struct intel_engine_cs *ring, + const struct drm_i915_cmd_table *cmd_tables, + int cmd_table_count) +{ + int i; + bool ret = true; + + if (!cmd_tables || cmd_table_count == 0) + return true; + + for (i = 0; i < cmd_table_count; i++) { + const struct drm_i915_cmd_table *table = &cmd_tables[i]; + u32 previous = 0; + int j; + + for (j = 0; j < table->count; j++) { + const struct drm_i915_cmd_descriptor *desc = + &table->table[i]; + u32 curr = desc->cmd.value & desc->cmd.mask; + + if (curr < previous) { + DRM_ERROR("CMD: table not sorted ring=%d table=%d entry=%d cmd=0x%08X prev=0x%08X\n", + ring->id, i, j, curr, previous); + ret = false; + } + + previous = curr; + } + } + + return ret; +} + +static bool check_sorted(int ring_id, const u32 *reg_table, int reg_count) +{ + int i; + u32 previous = 0; + bool ret = true; + + for (i = 0; i < reg_count; i++) { + u32 curr = reg_table[i]; + + if (curr < previous) { + DRM_ERROR("CMD: table not sorted ring=%d entry=%d reg=0x%08X prev=0x%08X\n", + ring_id, i, curr, previous); + ret = false; + } + + previous = curr; + } + + return ret; +} + +static bool validate_regs_sorted(struct intel_engine_cs *ring) +{ + return check_sorted(ring->id, ring->reg_table, ring->reg_count) && + check_sorted(ring->id, ring->master_reg_table, + ring->master_reg_count); +} + +struct cmd_node { + const struct drm_i915_cmd_descriptor *desc; + struct hlist_node node; +}; + +/* + * Different command ranges have different numbers of bits for the opcode. For + * example, MI commands use bits 31:23 while 3D commands use bits 31:16. The + * problem is that, for example, MI commands use bits 22:16 for other fields + * such as GGTT vs PPGTT bits. If we include those bits in the mask then when + * we mask a command from a batch it could hash to the wrong bucket due to + * non-opcode bits being set. But if we don't include those bits, some 3D + * commands may hash to the same bucket due to not including opcode bits that + * make the command unique. For now, we will risk hashing to the same bucket. + * + * If we attempt to generate a perfect hash, we should be able to look at bits + * 31:29 of a command from a batch buffer and use the full mask for that + * client. The existing INSTR_CLIENT_MASK/SHIFT defines can be used for this. + */ +#define CMD_HASH_MASK STD_MI_OPCODE_MASK + +static int init_hash_table(struct intel_engine_cs *ring, + const struct drm_i915_cmd_table *cmd_tables, + int cmd_table_count) +{ + int i, j; + + hash_init(ring->cmd_hash); + + for (i = 0; i < cmd_table_count; i++) { + const struct drm_i915_cmd_table *table = &cmd_tables[i]; + + for (j = 0; j < table->count; j++) { + const struct drm_i915_cmd_descriptor *desc = + &table->table[j]; + struct cmd_node *desc_node = + kmalloc(sizeof(*desc_node), GFP_KERNEL); + + if (!desc_node) + return -ENOMEM; + + desc_node->desc = desc; + hash_add(ring->cmd_hash, &desc_node->node, + desc->cmd.value & CMD_HASH_MASK); + } + } + + return 0; +} + +static void fini_hash_table(struct intel_engine_cs *ring) +{ + struct hlist_node *tmp; + struct cmd_node *desc_node; + int i; + + hash_for_each_safe(ring->cmd_hash, i, tmp, desc_node, node) { + hash_del(&desc_node->node); + kfree(desc_node); + } +} + +/** + * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer + * @ring: the ringbuffer to initialize + * + * Optionally initializes fields related to batch buffer command parsing in the + * struct intel_engine_cs based on whether the platform requires software + * command parsing. + * + * Return: non-zero if initialization fails + */ +int i915_cmd_parser_init_ring(struct intel_engine_cs *ring) +{ + const struct drm_i915_cmd_table *cmd_tables; + int cmd_table_count; + int ret; + + if (!IS_GEN7(ring->dev)) + return 0; + + switch (ring->id) { + case RCS: + if (IS_HASWELL(ring->dev)) { + cmd_tables = hsw_render_ring_cmds; + cmd_table_count = + ARRAY_SIZE(hsw_render_ring_cmds); + } else { + cmd_tables = gen7_render_cmds; + cmd_table_count = ARRAY_SIZE(gen7_render_cmds); + } + + ring->reg_table = gen7_render_regs; + ring->reg_count = ARRAY_SIZE(gen7_render_regs); + + if (IS_HASWELL(ring->dev)) { + ring->master_reg_table = hsw_master_regs; + ring->master_reg_count = ARRAY_SIZE(hsw_master_regs); + } else { + ring->master_reg_table = ivb_master_regs; + ring->master_reg_count = ARRAY_SIZE(ivb_master_regs); + } + + ring->get_cmd_length_mask = gen7_render_get_cmd_length_mask; + break; + case VCS: + cmd_tables = gen7_video_cmds; + cmd_table_count = ARRAY_SIZE(gen7_video_cmds); + ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; + break; + case BCS: + if (IS_HASWELL(ring->dev)) { + cmd_tables = hsw_blt_ring_cmds; + cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds); + } else { + cmd_tables = gen7_blt_cmds; + cmd_table_count = ARRAY_SIZE(gen7_blt_cmds); + } + + ring->reg_table = gen7_blt_regs; + ring->reg_count = ARRAY_SIZE(gen7_blt_regs); + + if (IS_HASWELL(ring->dev)) { + ring->master_reg_table = hsw_master_regs; + ring->master_reg_count = ARRAY_SIZE(hsw_master_regs); + } else { + ring->master_reg_table = ivb_master_regs; + ring->master_reg_count = ARRAY_SIZE(ivb_master_regs); + } + + ring->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; + break; + case VECS: + cmd_tables = hsw_vebox_cmds; + cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds); + /* VECS can use the same length_mask function as VCS */ + ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; + break; + default: + DRM_ERROR("CMD: cmd_parser_init with unknown ring: %d\n", + ring->id); + BUG(); + } + + BUG_ON(!validate_cmds_sorted(ring, cmd_tables, cmd_table_count)); + BUG_ON(!validate_regs_sorted(ring)); + + ret = init_hash_table(ring, cmd_tables, cmd_table_count); + if (ret) { + DRM_ERROR("CMD: cmd_parser_init failed!\n"); + fini_hash_table(ring); + return ret; + } + + ring->needs_cmd_parser = true; + + return 0; +} + +/** + * i915_cmd_parser_fini_ring() - clean up cmd parser related fields + * @ring: the ringbuffer to clean up + * + * Releases any resources related to command parsing that may have been + * initialized for the specified ring. + */ +void i915_cmd_parser_fini_ring(struct intel_engine_cs *ring) +{ + if (!ring->needs_cmd_parser) + return; + + fini_hash_table(ring); +} + +static const struct drm_i915_cmd_descriptor* +find_cmd_in_table(struct intel_engine_cs *ring, + u32 cmd_header) +{ + struct cmd_node *desc_node; + + hash_for_each_possible(ring->cmd_hash, desc_node, node, + cmd_header & CMD_HASH_MASK) { + const struct drm_i915_cmd_descriptor *desc = desc_node->desc; + u32 masked_cmd = desc->cmd.mask & cmd_header; + u32 masked_value = desc->cmd.value & desc->cmd.mask; + + if (masked_cmd == masked_value) + return desc; + } + + return NULL; +} + +/* + * Returns a pointer to a descriptor for the command specified by cmd_header. + * + * The caller must supply space for a default descriptor via the default_desc + * parameter. If no descriptor for the specified command exists in the ring's + * command parser tables, this function fills in default_desc based on the + * ring's default length encoding and returns default_desc. + */ +static const struct drm_i915_cmd_descriptor* +find_cmd(struct intel_engine_cs *ring, + u32 cmd_header, + struct drm_i915_cmd_descriptor *default_desc) +{ + const struct drm_i915_cmd_descriptor *desc; + u32 mask; + + desc = find_cmd_in_table(ring, cmd_header); + if (desc) + return desc; + + mask = ring->get_cmd_length_mask(cmd_header); + if (!mask) + return NULL; + + BUG_ON(!default_desc); + default_desc->flags = CMD_DESC_SKIP; + default_desc->length.mask = mask; + + return default_desc; +} + +static bool valid_reg(const u32 *table, int count, u32 addr) +{ + if (table && count != 0) { + int i; + + for (i = 0; i < count; i++) { + if (table[i] == addr) + return true; + } + } + + return false; +} + +static u32 *vmap_batch(struct drm_i915_gem_object *obj) +{ + int i; + void *addr = NULL; + struct sg_page_iter sg_iter; + struct page **pages; + + pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages)); + if (pages == NULL) { + DRM_DEBUG_DRIVER("Failed to get space for pages\n"); + goto finish; + } + + i = 0; + for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { + pages[i] = sg_page_iter_page(&sg_iter); + i++; + } + + addr = vmap(pages, i, 0, PAGE_KERNEL); + if (addr == NULL) { + DRM_DEBUG_DRIVER("Failed to vmap pages\n"); + goto finish; + } + +finish: + if (pages) + drm_free_large(pages); + return (u32*)addr; +} + +/** + * i915_needs_cmd_parser() - should a given ring use software command parsing? + * @ring: the ring in question + * + * Only certain platforms require software batch buffer command parsing, and + * only when enabled via module paramter. + * + * Return: true if the ring requires software command parsing + */ +bool i915_needs_cmd_parser(struct intel_engine_cs *ring) +{ + struct drm_i915_private *dev_priv = ring->dev->dev_private; + + if (!ring->needs_cmd_parser) + return false; + + /* + * XXX: VLV is Gen7 and therefore has cmd_tables, but has PPGTT + * disabled. That will cause all of the parser's PPGTT checks to + * fail. For now, disable parsing when PPGTT is off. + */ + if (!dev_priv->mm.aliasing_ppgtt) + return false; + + return (i915.enable_cmd_parser == 1); +} + +static bool check_cmd(const struct intel_engine_cs *ring, + const struct drm_i915_cmd_descriptor *desc, + const u32 *cmd, + const bool is_master, + bool *oacontrol_set) +{ + if (desc->flags & CMD_DESC_REJECT) { + DRM_DEBUG_DRIVER("CMD: Rejected command: 0x%08X\n", *cmd); + return false; + } + + if ((desc->flags & CMD_DESC_MASTER) && !is_master) { + DRM_DEBUG_DRIVER("CMD: Rejected master-only command: 0x%08X\n", + *cmd); + return false; + } + + if (desc->flags & CMD_DESC_REGISTER) { + u32 reg_addr = cmd[desc->reg.offset] & desc->reg.mask; + + /* + * OACONTROL requires some special handling for writes. We + * want to make sure that any batch which enables OA also + * disables it before the end of the batch. The goal is to + * prevent one process from snooping on the perf data from + * another process. To do that, we need to check the value + * that will be written to the register. Hence, limit + * OACONTROL writes to only MI_LOAD_REGISTER_IMM commands. + */ + if (reg_addr == OACONTROL) { + if (desc->cmd.value == MI_LOAD_REGISTER_MEM) + return false; + + if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1)) + *oacontrol_set = (cmd[2] != 0); + } + + if (!valid_reg(ring->reg_table, + ring->reg_count, reg_addr)) { + if (!is_master || + !valid_reg(ring->master_reg_table, + ring->master_reg_count, + reg_addr)) { + DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (ring=%d)\n", + reg_addr, + *cmd, + ring->id); + return false; + } + } + } + + if (desc->flags & CMD_DESC_BITMASK) { + int i; + + for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) { + u32 dword; + + if (desc->bits[i].mask == 0) + break; + + if (desc->bits[i].condition_mask != 0) { + u32 offset = + desc->bits[i].condition_offset; + u32 condition = cmd[offset] & + desc->bits[i].condition_mask; + + if (condition == 0) + continue; + } + + dword = cmd[desc->bits[i].offset] & + desc->bits[i].mask; + + if (dword != desc->bits[i].expected) { + DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (ring=%d)\n", + *cmd, + desc->bits[i].mask, + desc->bits[i].expected, + dword, ring->id); + return false; + } + } + } + + return true; +} + +#define LENGTH_BIAS 2 + +/** + * i915_parse_cmds() - parse a submitted batch buffer for privilege violations + * @ring: the ring on which the batch is to execute + * @batch_obj: the batch buffer in question + * @batch_start_offset: byte offset in the batch at which execution starts + * @is_master: is the submitting process the drm master? + * + * Parses the specified batch buffer looking for privilege violations as + * described in the overview. + * + * Return: non-zero if the parser finds violations or otherwise fails + */ +int i915_parse_cmds(struct intel_engine_cs *ring, + struct drm_i915_gem_object *batch_obj, + u32 batch_start_offset, + bool is_master) +{ + int ret = 0; + u32 *cmd, *batch_base, *batch_end; + struct drm_i915_cmd_descriptor default_desc = { 0 }; + int needs_clflush = 0; + bool oacontrol_set = false; /* OACONTROL tracking. See check_cmd() */ + + ret = i915_gem_obj_prepare_shmem_read(batch_obj, &needs_clflush); + if (ret) { + DRM_DEBUG_DRIVER("CMD: failed to prep read\n"); + return ret; + } + + batch_base = vmap_batch(batch_obj); + if (!batch_base) { + DRM_DEBUG_DRIVER("CMD: Failed to vmap batch\n"); + i915_gem_object_unpin_pages(batch_obj); + return -ENOMEM; + } + + if (needs_clflush) + drm_clflush_virt_range((char *)batch_base, batch_obj->base.size); + + cmd = batch_base + (batch_start_offset / sizeof(*cmd)); + batch_end = cmd + (batch_obj->base.size / sizeof(*batch_end)); + + while (cmd < batch_end) { + const struct drm_i915_cmd_descriptor *desc; + u32 length; + + if (*cmd == MI_BATCH_BUFFER_END) + break; + + desc = find_cmd(ring, *cmd, &default_desc); + if (!desc) { + DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n", + *cmd); + ret = -EINVAL; + break; + } + + if (desc->flags & CMD_DESC_FIXED) + length = desc->length.fixed; + else + length = ((*cmd & desc->length.mask) + LENGTH_BIAS); + + if ((batch_end - cmd) < length) { + DRM_DEBUG_DRIVER("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n", + *cmd, + length, + batch_end - cmd); + ret = -EINVAL; + break; + } + + if (!check_cmd(ring, desc, cmd, is_master, &oacontrol_set)) { + ret = -EINVAL; + break; + } + + cmd += length; + } + + if (oacontrol_set) { + DRM_DEBUG_DRIVER("CMD: batch set OACONTROL but did not clear it\n"); + ret = -EINVAL; + } + + if (cmd >= batch_end) { + DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n"); + ret = -EINVAL; + } + + vunmap(batch_base); + + i915_gem_object_unpin_pages(batch_obj); + + return ret; +} + +/** + * i915_cmd_parser_get_version() - get the cmd parser version number + * + * The cmd parser maintains a simple increasing integer version number suitable + * for passing to userspace clients to determine what operations are permitted. + * + * Return: the current version number of the cmd parser + */ +int i915_cmd_parser_get_version(void) +{ + /* + * Command parser version history + * + * 1. Initial version. Checks batches and reports violations, but leaves + * hardware parsing enabled (so does not allow new use cases). + */ + return 1; +} diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index b2b46c52294..b8c689202c4 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -79,7 +79,7 @@ drm_add_fake_info_node(struct drm_minor *minor, static int i915_capabilities(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; const struct intel_device_info *info = INTEL_INFO(dev); @@ -98,7 +98,7 @@ static const char *get_pin_flag(struct drm_i915_gem_object *obj) { if (obj->user_pin_count > 0) return "P"; - else if (obj->pin_count > 0) + else if (i915_gem_obj_is_pinned(obj)) return "p"; else return " "; @@ -123,6 +123,8 @@ static void describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { struct i915_vma *vma; + int pin_count = 0; + seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %u %u %u%s%s%s", &obj->base, get_pin_flag(obj), @@ -139,8 +141,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) seq_printf(m, " (name: %d)", obj->base.name); - if (obj->pin_count) - seq_printf(m, " (pinned x %d)", obj->pin_count); + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (vma->pin_count > 0) + pin_count++; + seq_printf(m, " (pinned x %d)", pin_count); if (obj->pin_display) seq_printf(m, " (display)"); if (obj->fence_reg != I915_FENCE_REG_NONE) @@ -168,7 +172,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) seq_printf(m, " (%s)", obj->ring->name); } -static void describe_ctx(struct seq_file *m, struct i915_hw_context *ctx) +static void describe_ctx(struct seq_file *m, struct intel_context *ctx) { seq_putc(m, ctx->is_initialized ? 'I' : 'i'); seq_putc(m, ctx->remap_slice ? 'R' : 'r'); @@ -177,7 +181,7 @@ static void describe_ctx(struct seq_file *m, struct i915_hw_context *ctx) static int i915_gem_object_list_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; uintptr_t list = (uintptr_t) node->info_ent->data; struct list_head *head; struct drm_device *dev = node->minor->dev; @@ -235,7 +239,7 @@ static int obj_rank_by_stolen(void *priv, static int i915_gem_stolen_list_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; @@ -295,28 +299,62 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) } while (0) struct file_stats { + struct drm_i915_file_private *file_priv; int count; - size_t total, active, inactive, unbound; + size_t total, unbound; + size_t global, shared; + size_t active, inactive; }; static int per_file_stats(int id, void *ptr, void *data) { struct drm_i915_gem_object *obj = ptr; struct file_stats *stats = data; + struct i915_vma *vma; stats->count++; stats->total += obj->base.size; - if (i915_gem_obj_ggtt_bound(obj)) { - if (!list_empty(&obj->ring_list)) - stats->active += obj->base.size; - else - stats->inactive += obj->base.size; + if (obj->base.name || obj->base.dma_buf) + stats->shared += obj->base.size; + + if (USES_FULL_PPGTT(obj->base.dev)) { + list_for_each_entry(vma, &obj->vma_list, vma_link) { + struct i915_hw_ppgtt *ppgtt; + + if (!drm_mm_node_allocated(&vma->node)) + continue; + + if (i915_is_ggtt(vma->vm)) { + stats->global += obj->base.size; + continue; + } + + ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base); + if (ppgtt->ctx && ppgtt->ctx->file_priv != stats->file_priv) + continue; + + if (obj->ring) /* XXX per-vma statistic */ + stats->active += obj->base.size; + else + stats->inactive += obj->base.size; + + return 0; + } } else { - if (!list_empty(&obj->global_list)) - stats->unbound += obj->base.size; + if (i915_gem_obj_ggtt_bound(obj)) { + stats->global += obj->base.size; + if (obj->ring) + stats->active += obj->base.size; + else + stats->inactive += obj->base.size; + return 0; + } } + if (!list_empty(&obj->global_list)) + stats->unbound += obj->base.size; + return 0; } @@ -333,7 +371,7 @@ static int per_file_stats(int id, void *ptr, void *data) static int i915_gem_object_info(struct seq_file *m, void* data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 count, mappable_count, purgeable_count; @@ -407,7 +445,10 @@ static int i915_gem_object_info(struct seq_file *m, void* data) struct task_struct *task; memset(&stats, 0, sizeof(stats)); + stats.file_priv = file->driver_priv; + spin_lock(&file->table_lock); idr_for_each(&file->object_idr, per_file_stats, &stats); + spin_unlock(&file->table_lock); /* * Although we have a valid reference on file->pid, that does * not guarantee that the task_struct who called get_pid() is @@ -416,12 +457,14 @@ static int i915_gem_object_info(struct seq_file *m, void* data) */ rcu_read_lock(); task = pid_task(file->pid, PIDTYPE_PID); - seq_printf(m, "%s: %u objects, %zu bytes (%zu active, %zu inactive, %zu unbound)\n", + seq_printf(m, "%s: %u objects, %zu bytes (%zu active, %zu inactive, %zu global, %zu shared, %zu unbound)\n", task ? task->comm : "<unknown>", stats.count, stats.total, stats.active, stats.inactive, + stats.global, + stats.shared, stats.unbound); rcu_read_unlock(); } @@ -433,7 +476,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data) static int i915_gem_gtt_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; uintptr_t list = (uintptr_t) node->info_ent->data; struct drm_i915_private *dev_priv = dev->dev_private; @@ -447,7 +490,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) total_obj_size = total_gtt_size = count = 0; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { - if (list == PINNED_LIST && obj->pin_count == 0) + if (list == PINNED_LIST && !i915_gem_obj_is_pinned(obj)) continue; seq_puts(m, " "); @@ -468,12 +511,12 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) static int i915_gem_pageflip_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; unsigned long flags; struct intel_crtc *crtc; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) { + for_each_intel_crtc(dev, crtc) { const char pipe = pipe_name(crtc->pipe); const char plane = plane_name(crtc->plane); struct intel_unpin_work *work; @@ -518,10 +561,10 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) static int i915_gem_request_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; struct drm_i915_gem_request *gem_request; int ret, count, i; @@ -553,7 +596,7 @@ static int i915_gem_request_info(struct seq_file *m, void *data) } static void i915_ring_seqno_info(struct seq_file *m, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { if (ring->get_seqno) { seq_printf(m, "Current sequence (%s): %u\n", @@ -563,10 +606,10 @@ static void i915_ring_seqno_info(struct seq_file *m, static int i915_gem_seqno_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; int ret, i; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -586,10 +629,10 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data) static int i915_interrupt_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; int ret, i, pipe; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -597,11 +640,31 @@ static int i915_interrupt_info(struct seq_file *m, void *data) return ret; intel_runtime_pm_get(dev_priv); - if (INTEL_INFO(dev)->gen >= 8) { + if (IS_CHERRYVIEW(dev)) { int i; seq_printf(m, "Master Interrupt Control:\t%08x\n", I915_READ(GEN8_MASTER_IRQ)); + seq_printf(m, "Display IER:\t%08x\n", + I915_READ(VLV_IER)); + seq_printf(m, "Display IIR:\t%08x\n", + I915_READ(VLV_IIR)); + seq_printf(m, "Display IIR_RW:\t%08x\n", + I915_READ(VLV_IIR_RW)); + seq_printf(m, "Display IMR:\t%08x\n", + I915_READ(VLV_IMR)); + for_each_pipe(pipe) + seq_printf(m, "Pipe %c stat:\t%08x\n", + pipe_name(pipe), + I915_READ(PIPESTAT(pipe))); + + seq_printf(m, "Port hotplug:\t%08x\n", + I915_READ(PORT_HOTPLUG_EN)); + seq_printf(m, "DPFLIPSTAT:\t%08x\n", + I915_READ(VLV_DPFLIPSTAT)); + seq_printf(m, "DPINVGTT:\t%08x\n", + I915_READ(DPINVGTT)); + for (i = 0; i < 4; i++) { seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", i, I915_READ(GEN8_GT_IMR(i))); @@ -611,16 +674,35 @@ static int i915_interrupt_info(struct seq_file *m, void *data) i, I915_READ(GEN8_GT_IER(i))); } - for_each_pipe(i) { + seq_printf(m, "PCU interrupt mask:\t%08x\n", + I915_READ(GEN8_PCU_IMR)); + seq_printf(m, "PCU interrupt identity:\t%08x\n", + I915_READ(GEN8_PCU_IIR)); + seq_printf(m, "PCU interrupt enable:\t%08x\n", + I915_READ(GEN8_PCU_IER)); + } else if (INTEL_INFO(dev)->gen >= 8) { + seq_printf(m, "Master Interrupt Control:\t%08x\n", + I915_READ(GEN8_MASTER_IRQ)); + + for (i = 0; i < 4; i++) { + seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", + i, I915_READ(GEN8_GT_IMR(i))); + seq_printf(m, "GT Interrupt IIR %d:\t%08x\n", + i, I915_READ(GEN8_GT_IIR(i))); + seq_printf(m, "GT Interrupt IER %d:\t%08x\n", + i, I915_READ(GEN8_GT_IER(i))); + } + + for_each_pipe(pipe) { seq_printf(m, "Pipe %c IMR:\t%08x\n", - pipe_name(i), - I915_READ(GEN8_DE_PIPE_IMR(i))); + pipe_name(pipe), + I915_READ(GEN8_DE_PIPE_IMR(pipe))); seq_printf(m, "Pipe %c IIR:\t%08x\n", - pipe_name(i), - I915_READ(GEN8_DE_PIPE_IIR(i))); + pipe_name(pipe), + I915_READ(GEN8_DE_PIPE_IIR(pipe))); seq_printf(m, "Pipe %c IER:\t%08x\n", - pipe_name(i), - I915_READ(GEN8_DE_PIPE_IER(i))); + pipe_name(pipe), + I915_READ(GEN8_DE_PIPE_IER(pipe))); } seq_printf(m, "Display Engine port interrupt mask:\t%08x\n", @@ -712,8 +794,6 @@ static int i915_interrupt_info(struct seq_file *m, void *data) seq_printf(m, "Graphics Interrupt mask: %08x\n", I915_READ(GTIMR)); } - seq_printf(m, "Interrupts received: %d\n", - atomic_read(&dev_priv->irq_received)); for_each_ring(ring, dev_priv, i) { if (INTEL_INFO(dev)->gen >= 6) { seq_printf(m, @@ -730,9 +810,9 @@ static int i915_interrupt_info(struct seq_file *m, void *data) static int i915_gem_fence_regs_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int i, ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -759,10 +839,10 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data) static int i915_hws_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; const u32 *hws; int i; @@ -872,7 +952,7 @@ static int i915_next_seqno_get(void *data, u64 *val) { struct drm_device *dev = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -907,9 +987,9 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_next_seqno_fops, static int i915_rstdby_delays(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u16 crstanddelay; int ret; @@ -928,11 +1008,11 @@ static int i915_rstdby_delays(struct seq_file *m, void *unused) return 0; } -static int i915_cur_delayinfo(struct seq_file *m, void *unused) +static int i915_frequency_info(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret = 0; intel_runtime_pm_get(dev_priv); @@ -953,6 +1033,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused) u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS); u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + u32 rpmodectl, rpinclimit, rpdeclimit; u32 rpstat, cagf, reqf; u32 rpupei, rpcurup, rpprevup; u32 rpdownei, rpcurdown, rpprevdown; @@ -973,6 +1054,10 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused) reqf >>= 25; reqf *= GT_FREQUENCY_MULTIPLIER; + rpmodectl = I915_READ(GEN6_RP_CONTROL); + rpinclimit = I915_READ(GEN6_RP_UP_THRESHOLD); + rpdeclimit = I915_READ(GEN6_RP_DOWN_THRESHOLD); + rpstat = I915_READ(GEN6_RPSTAT1); rpupei = I915_READ(GEN6_RP_CUR_UP_EI); rpcurup = I915_READ(GEN6_RP_CUR_UP); @@ -989,14 +1074,23 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused) gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); mutex_unlock(&dev->struct_mutex); + seq_printf(m, "PM IER=0x%08x IMR=0x%08x ISR=0x%08x IIR=0x%08x, MASK=0x%08x\n", + I915_READ(GEN6_PMIER), + I915_READ(GEN6_PMIMR), + I915_READ(GEN6_PMISR), + I915_READ(GEN6_PMIIR), + I915_READ(GEN6_PMINTRMSK)); seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); - seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat); seq_printf(m, "Render p-state ratio: %d\n", (gt_perf_status & 0xff00) >> 8); seq_printf(m, "Render p-state VID: %d\n", gt_perf_status & 0xff); seq_printf(m, "Render p-state limit: %d\n", rp_state_limits & 0xff); + seq_printf(m, "RPSTAT1: 0x%08x\n", rpstat); + seq_printf(m, "RPMODECTL: 0x%08x\n", rpmodectl); + seq_printf(m, "RPINCLIMIT: 0x%08x\n", rpinclimit); + seq_printf(m, "RPDECLIMIT: 0x%08x\n", rpdeclimit); seq_printf(m, "RPNSWREQ: %dMHz\n", reqf); seq_printf(m, "CAGF: %dMHz\n", cagf); seq_printf(m, "RP CUR UP EI: %dus\n", rpupei & @@ -1025,7 +1119,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused) max_freq * GT_FREQUENCY_MULTIPLIER); seq_printf(m, "Max overclocked frequency: %dMHz\n", - dev_priv->rps.hw_max * GT_FREQUENCY_MULTIPLIER); + dev_priv->rps.max_freq * GT_FREQUENCY_MULTIPLIER); } else if (IS_VALLEYVIEW(dev)) { u32 freq_sts, val; @@ -1056,9 +1150,9 @@ out: static int i915_delayfreq_table(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 delayfreq; int ret, i; @@ -1087,9 +1181,9 @@ static inline int MAP_TO_MV(int map) static int i915_inttoext_table(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 inttoext; int ret, i; @@ -1111,9 +1205,9 @@ static int i915_inttoext_table(struct seq_file *m, void *unused) static int ironlake_drpc_info(struct seq_file *m) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 rgvmodectl, rstdbyctl; u16 crstandvid; int ret; @@ -1181,15 +1275,19 @@ static int ironlake_drpc_info(struct seq_file *m) static int vlv_drpc_info(struct seq_file *m) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 rpmodectl1, rcctl1; unsigned fw_rendercount = 0, fw_mediacount = 0; + intel_runtime_pm_get(dev_priv); + rpmodectl1 = I915_READ(GEN6_RP_CONTROL); rcctl1 = I915_READ(GEN6_RC_CONTROL); + intel_runtime_pm_put(dev_priv); + seq_printf(m, "Video Turbo Mode: %s\n", yesno(rpmodectl1 & GEN6_RP_MEDIA_TURBO)); seq_printf(m, "Turbo enabled: %s\n", @@ -1209,6 +1307,11 @@ static int vlv_drpc_info(struct seq_file *m) (I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_PW_MEDIA_STATUS_MASK) ? "Up" : "Down"); + seq_printf(m, "Render RC6 residency since boot: %u\n", + I915_READ(VLV_GT_RENDER_RC6)); + seq_printf(m, "Media RC6 residency since boot: %u\n", + I915_READ(VLV_GT_MEDIA_RC6)); + spin_lock_irq(&dev_priv->uncore.lock); fw_rendercount = dev_priv->uncore.fw_rendercount; fw_mediacount = dev_priv->uncore.fw_mediacount; @@ -1225,7 +1328,7 @@ static int vlv_drpc_info(struct seq_file *m) static int gen6_drpc_info(struct seq_file *m) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 rpmodectl1, gt_core_status, rcctl1, rc6vids = 0; @@ -1324,7 +1427,7 @@ static int gen6_drpc_info(struct seq_file *m) static int i915_drpc_info(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; if (IS_VALLEYVIEW(dev)) @@ -1337,15 +1440,17 @@ static int i915_drpc_info(struct seq_file *m, void *unused) static int i915_fbc_status(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; if (!HAS_FBC(dev)) { seq_puts(m, "FBC unsupported on this chipset\n"); return 0; } + intel_runtime_pm_get(dev_priv); + if (intel_fbc_enabled(dev)) { seq_puts(m, "FBC enabled\n"); } else { @@ -1389,12 +1494,15 @@ static int i915_fbc_status(struct seq_file *m, void *unused) } seq_putc(m, '\n'); } + + intel_runtime_pm_put(dev_priv); + return 0; } static int i915_ips_status(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1403,21 +1511,27 @@ static int i915_ips_status(struct seq_file *m, void *unused) return 0; } + intel_runtime_pm_get(dev_priv); + if (IS_BROADWELL(dev) || I915_READ(IPS_CTL) & IPS_ENABLE) seq_puts(m, "enabled\n"); else seq_puts(m, "disabled\n"); + intel_runtime_pm_put(dev_priv); + return 0; } static int i915_sr_status(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; bool sr_enabled = false; + intel_runtime_pm_get(dev_priv); + if (HAS_PCH_SPLIT(dev)) sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev)) @@ -1427,6 +1541,8 @@ static int i915_sr_status(struct seq_file *m, void *unused) else if (IS_PINEVIEW(dev)) sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN; + intel_runtime_pm_put(dev_priv); + seq_printf(m, "self-refresh: %s\n", sr_enabled ? "enabled" : "disabled"); @@ -1435,9 +1551,9 @@ static int i915_sr_status(struct seq_file *m, void *unused) static int i915_emon_status(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long temp, chipset, gfx; int ret; @@ -1463,10 +1579,10 @@ static int i915_emon_status(struct seq_file *m, void *unused) static int i915_ring_freq_table(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - int ret; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret = 0; int gpu_freq, ia_freq; if (!(IS_GEN6(dev) || IS_GEN7(dev))) { @@ -1474,17 +1590,18 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) return 0; } + intel_runtime_pm_get(dev_priv); + flush_delayed_work(&dev_priv->rps.delayed_resume_work); ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock); if (ret) - return ret; - intel_runtime_pm_get(dev_priv); + goto out; seq_puts(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\tEffective Ring freq (MHz)\n"); - for (gpu_freq = dev_priv->rps.min_delay; - gpu_freq <= dev_priv->rps.max_delay; + for (gpu_freq = dev_priv->rps.min_freq_softlimit; + gpu_freq <= dev_priv->rps.max_freq_softlimit; gpu_freq++) { ia_freq = gpu_freq; sandybridge_pcode_read(dev_priv, @@ -1496,17 +1613,18 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) ((ia_freq >> 8) & 0xff) * 100); } - intel_runtime_pm_put(dev_priv); mutex_unlock(&dev_priv->rps.hw_lock); - return 0; +out: + intel_runtime_pm_put(dev_priv); + return ret; } static int i915_gfxec(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -1524,9 +1642,9 @@ static int i915_gfxec(struct seq_file *m, void *unused) static int i915_opregion(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; void *data = kmalloc(OPREGION_SIZE, GFP_KERNEL); int ret; @@ -1552,7 +1670,7 @@ out: static int i915_gem_framebuffer_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct intel_fbdev *ifbdev = NULL; struct intel_framebuffer *fb; @@ -1598,11 +1716,11 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) static int i915_context_status(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; - struct i915_hw_context *ctx; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; + struct intel_context *ctx; int ret, i; ret = mutex_lock_interruptible(&dev->mode_config.mutex); @@ -1622,6 +1740,9 @@ static int i915_context_status(struct seq_file *m, void *unused) } list_for_each_entry(ctx, &dev_priv->context_list, link) { + if (ctx->obj == NULL) + continue; + seq_puts(m, "HW context "); describe_ctx(m, ctx); for_each_ring(ring, dev_priv, i) @@ -1639,7 +1760,7 @@ static int i915_context_status(struct seq_file *m, void *unused) static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; unsigned forcewake_count = 0, fw_rendercount = 0, fw_mediacount = 0; @@ -1687,7 +1808,7 @@ static const char *swizzle_string(unsigned swizzle) static int i915_swizzle_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; int ret; @@ -1733,10 +1854,25 @@ static int i915_swizzle_info(struct seq_file *m, void *data) return 0; } +static int per_file_ctx(int id, void *ptr, void *data) +{ + struct intel_context *ctx = ptr; + struct seq_file *m = data; + struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx); + + if (i915_gem_context_is_default(ctx)) + seq_puts(m, " default context:\n"); + else + seq_printf(m, " context %d:\n", ctx->id); + ppgtt->debug_dump(ppgtt, m); + + return 0; +} + static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; int unused, i; @@ -1744,7 +1880,7 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) return; seq_printf(m, "Page directories: %d\n", ppgtt->num_pd_pages); - seq_printf(m, "Page tables: %d\n", ppgtt->num_pt_pages); + seq_printf(m, "Page tables: %d\n", ppgtt->num_pd_entries); for_each_ring(ring, dev_priv, unused) { seq_printf(m, "%s\n", ring->name); for (i = 0; i < 4; i++) { @@ -1752,8 +1888,7 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) u64 pdp = I915_READ(ring->mmio_base + offset + 4); pdp <<= 32; pdp |= I915_READ(ring->mmio_base + offset); - for (i = 0; i < 4; i++) - seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp); + seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp); } } } @@ -1761,7 +1896,8 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; + struct drm_file *file; int i; if (INTEL_INFO(dev)->gen == 6) @@ -1780,13 +1916,24 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) seq_puts(m, "aliasing PPGTT:\n"); seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset); + + ppgtt->debug_dump(ppgtt, m); + } else + return; + + list_for_each_entry_reverse(file, &dev->filelist, lhead) { + struct drm_i915_file_private *file_priv = file->driver_priv; + + seq_printf(m, "proc: %s\n", + get_pid_task(file->pid, PIDTYPE_PID)->comm); + idr_for_each(&file_priv->context_idr, per_file_ctx, m); } seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK)); } static int i915_ppgtt_info(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1806,56 +1953,9 @@ static int i915_ppgtt_info(struct seq_file *m, void *data) return 0; } -static int i915_dpio_info(struct seq_file *m, void *data) -{ - struct drm_info_node *node = (struct drm_info_node *) m->private; - struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - - - if (!IS_VALLEYVIEW(dev)) { - seq_puts(m, "unsupported\n"); - return 0; - } - - ret = mutex_lock_interruptible(&dev_priv->dpio_lock); - if (ret) - return ret; - - seq_printf(m, "DPIO_CTL: 0x%08x\n", I915_READ(DPIO_CTL)); - - seq_printf(m, "DPIO PLL DW3 CH0 : 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW3(0))); - seq_printf(m, "DPIO PLL DW3 CH1: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW3(1))); - - seq_printf(m, "DPIO PLL DW5 CH0: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW5(0))); - seq_printf(m, "DPIO PLL DW5 CH1: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW5(1))); - - seq_printf(m, "DPIO PLL DW7 CH0: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW7(0))); - seq_printf(m, "DPIO PLL DW7 CH1: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW7(1))); - - seq_printf(m, "DPIO PLL DW10 CH0: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW10(0))); - seq_printf(m, "DPIO PLL DW10 CH1: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_PLL_DW10(1))); - - seq_printf(m, "DPIO_FASTCLK_DISABLE: 0x%08x\n", - vlv_dpio_read(dev_priv, PIPE_A, VLV_CMN_DW0)); - - mutex_unlock(&dev_priv->dpio_lock); - - return 0; -} - static int i915_llc(struct seq_file *m, void *data) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1892,6 +1992,47 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) return 0; } +static int i915_sink_crc(struct seq_file *m, void *data) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct intel_encoder *encoder; + struct intel_connector *connector; + struct intel_dp *intel_dp = NULL; + int ret; + u8 crc[6]; + + drm_modeset_lock_all(dev); + list_for_each_entry(connector, &dev->mode_config.connector_list, + base.head) { + + if (connector->base.dpms != DRM_MODE_DPMS_ON) + continue; + + if (!connector->base.encoder) + continue; + + encoder = to_intel_encoder(connector->base.encoder); + if (encoder->type != INTEL_OUTPUT_EDP) + continue; + + intel_dp = enc_to_intel_dp(&encoder->base); + + ret = intel_dp_sink_crc(intel_dp, crc); + if (ret) + goto out; + + seq_printf(m, "%02x%02x%02x%02x%02x%02x\n", + crc[0], crc[1], crc[2], + crc[3], crc[4], crc[5]); + goto out; + } + ret = -ENODEV; +out: + drm_modeset_unlock_all(dev); + return ret; +} + static int i915_energy_uJ(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; @@ -1903,12 +2044,16 @@ static int i915_energy_uJ(struct seq_file *m, void *data) if (INTEL_INFO(dev)->gen < 6) return -ENODEV; + intel_runtime_pm_get(dev_priv); + rdmsrl(MSR_RAPL_POWER_UNIT, power); power = (power & 0x1f00) >> 8; units = 1000000 / (1 << power); /* convert to uJ */ power = I915_READ(MCH_SECP_NRG_STTS); power *= units; + intel_runtime_pm_put(dev_priv); + seq_printf(m, "%llu", (long long unsigned)power); return 0; @@ -1916,24 +2061,18 @@ static int i915_energy_uJ(struct seq_file *m, void *data) static int i915_pc8_status(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - if (!IS_HASWELL(dev)) { + if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) { seq_puts(m, "not supported\n"); return 0; } - mutex_lock(&dev_priv->pc8.lock); - seq_printf(m, "Requirements met: %s\n", - yesno(dev_priv->pc8.requirements_met)); - seq_printf(m, "GPU idle: %s\n", yesno(dev_priv->pc8.gpu_idle)); - seq_printf(m, "Disable count: %d\n", dev_priv->pc8.disable_count); + seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->mm.busy)); seq_printf(m, "IRQs disabled: %s\n", - yesno(dev_priv->pc8.irqs_disabled)); - seq_printf(m, "Enabled: %s\n", yesno(dev_priv->pc8.enabled)); - mutex_unlock(&dev_priv->pc8.lock); + yesno(dev_priv->pm.irqs_disabled)); return 0; } @@ -1961,6 +2100,28 @@ static const char *power_domain_str(enum intel_display_power_domain domain) return "TRANSCODER_C"; case POWER_DOMAIN_TRANSCODER_EDP: return "TRANSCODER_EDP"; + case POWER_DOMAIN_PORT_DDI_A_2_LANES: + return "PORT_DDI_A_2_LANES"; + case POWER_DOMAIN_PORT_DDI_A_4_LANES: + return "PORT_DDI_A_4_LANES"; + case POWER_DOMAIN_PORT_DDI_B_2_LANES: + return "PORT_DDI_B_2_LANES"; + case POWER_DOMAIN_PORT_DDI_B_4_LANES: + return "PORT_DDI_B_4_LANES"; + case POWER_DOMAIN_PORT_DDI_C_2_LANES: + return "PORT_DDI_C_2_LANES"; + case POWER_DOMAIN_PORT_DDI_C_4_LANES: + return "PORT_DDI_C_4_LANES"; + case POWER_DOMAIN_PORT_DDI_D_2_LANES: + return "PORT_DDI_D_2_LANES"; + case POWER_DOMAIN_PORT_DDI_D_4_LANES: + return "PORT_DDI_D_4_LANES"; + case POWER_DOMAIN_PORT_DSI: + return "PORT_DSI"; + case POWER_DOMAIN_PORT_CRT: + return "PORT_CRT"; + case POWER_DOMAIN_PORT_OTHER: + return "PORT_OTHER"; case POWER_DOMAIN_VGA: return "VGA"; case POWER_DOMAIN_AUDIO: @@ -1975,7 +2136,7 @@ static const char *power_domain_str(enum intel_display_power_domain domain) static int i915_power_domain_info(struct seq_file *m, void *unused) { - struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; @@ -2008,6 +2169,214 @@ static int i915_power_domain_info(struct seq_file *m, void *unused) return 0; } +static void intel_seq_print_mode(struct seq_file *m, int tabs, + struct drm_display_mode *mode) +{ + int i; + + for (i = 0; i < tabs; i++) + seq_putc(m, '\t'); + + seq_printf(m, "id %d:\"%s\" freq %d clock %d hdisp %d hss %d hse %d htot %d vdisp %d vss %d vse %d vtot %d type 0x%x flags 0x%x\n", + mode->base.id, mode->name, + mode->vrefresh, mode->clock, + mode->hdisplay, mode->hsync_start, + mode->hsync_end, mode->htotal, + mode->vdisplay, mode->vsync_start, + mode->vsync_end, mode->vtotal, + mode->type, mode->flags); +} + +static void intel_encoder_info(struct seq_file *m, + struct intel_crtc *intel_crtc, + struct intel_encoder *intel_encoder) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_crtc *crtc = &intel_crtc->base; + struct intel_connector *intel_connector; + struct drm_encoder *encoder; + + encoder = &intel_encoder->base; + seq_printf(m, "\tencoder %d: type: %s, connectors:\n", + encoder->base.id, encoder->name); + for_each_connector_on_encoder(dev, encoder, intel_connector) { + struct drm_connector *connector = &intel_connector->base; + seq_printf(m, "\t\tconnector %d: type: %s, status: %s", + connector->base.id, + connector->name, + drm_get_connector_status_name(connector->status)); + if (connector->status == connector_status_connected) { + struct drm_display_mode *mode = &crtc->mode; + seq_printf(m, ", mode:\n"); + intel_seq_print_mode(m, 2, mode); + } else { + seq_putc(m, '\n'); + } + } +} + +static void intel_crtc_info(struct seq_file *m, struct intel_crtc *intel_crtc) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_crtc *crtc = &intel_crtc->base; + struct intel_encoder *intel_encoder; + + seq_printf(m, "\tfb: %d, pos: %dx%d, size: %dx%d\n", + crtc->primary->fb->base.id, crtc->x, crtc->y, + crtc->primary->fb->width, crtc->primary->fb->height); + for_each_encoder_on_crtc(dev, crtc, intel_encoder) + intel_encoder_info(m, intel_crtc, intel_encoder); +} + +static void intel_panel_info(struct seq_file *m, struct intel_panel *panel) +{ + struct drm_display_mode *mode = panel->fixed_mode; + + seq_printf(m, "\tfixed mode:\n"); + intel_seq_print_mode(m, 2, mode); +} + +static void intel_dp_info(struct seq_file *m, + struct intel_connector *intel_connector) +{ + struct intel_encoder *intel_encoder = intel_connector->encoder; + struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); + + seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]); + seq_printf(m, "\taudio support: %s\n", intel_dp->has_audio ? "yes" : + "no"); + if (intel_encoder->type == INTEL_OUTPUT_EDP) + intel_panel_info(m, &intel_connector->panel); +} + +static void intel_hdmi_info(struct seq_file *m, + struct intel_connector *intel_connector) +{ + struct intel_encoder *intel_encoder = intel_connector->encoder; + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); + + seq_printf(m, "\taudio support: %s\n", intel_hdmi->has_audio ? "yes" : + "no"); +} + +static void intel_lvds_info(struct seq_file *m, + struct intel_connector *intel_connector) +{ + intel_panel_info(m, &intel_connector->panel); +} + +static void intel_connector_info(struct seq_file *m, + struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + struct intel_encoder *intel_encoder = intel_connector->encoder; + struct drm_display_mode *mode; + + seq_printf(m, "connector %d: type %s, status: %s\n", + connector->base.id, connector->name, + drm_get_connector_status_name(connector->status)); + if (connector->status == connector_status_connected) { + seq_printf(m, "\tname: %s\n", connector->display_info.name); + seq_printf(m, "\tphysical dimensions: %dx%dmm\n", + connector->display_info.width_mm, + connector->display_info.height_mm); + seq_printf(m, "\tsubpixel order: %s\n", + drm_get_subpixel_order_name(connector->display_info.subpixel_order)); + seq_printf(m, "\tCEA rev: %d\n", + connector->display_info.cea_rev); + } + if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT || + intel_encoder->type == INTEL_OUTPUT_EDP) + intel_dp_info(m, intel_connector); + else if (intel_encoder->type == INTEL_OUTPUT_HDMI) + intel_hdmi_info(m, intel_connector); + else if (intel_encoder->type == INTEL_OUTPUT_LVDS) + intel_lvds_info(m, intel_connector); + + seq_printf(m, "\tmodes:\n"); + list_for_each_entry(mode, &connector->modes, head) + intel_seq_print_mode(m, 2, mode); +} + +static bool cursor_active(struct drm_device *dev, int pipe) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 state; + + if (IS_845G(dev) || IS_I865G(dev)) + state = I915_READ(_CURACNTR) & CURSOR_ENABLE; + else + state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE; + + return state; +} + +static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 pos; + + pos = I915_READ(CURPOS(pipe)); + + *x = (pos >> CURSOR_X_SHIFT) & CURSOR_POS_MASK; + if (pos & (CURSOR_POS_SIGN << CURSOR_X_SHIFT)) + *x = -*x; + + *y = (pos >> CURSOR_Y_SHIFT) & CURSOR_POS_MASK; + if (pos & (CURSOR_POS_SIGN << CURSOR_Y_SHIFT)) + *y = -*y; + + return cursor_active(dev, pipe); +} + +static int i915_display_info(struct seq_file *m, void *unused) +{ + struct drm_info_node *node = m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc; + struct drm_connector *connector; + + intel_runtime_pm_get(dev_priv); + drm_modeset_lock_all(dev); + seq_printf(m, "CRTC info\n"); + seq_printf(m, "---------\n"); + for_each_intel_crtc(dev, crtc) { + bool active; + int x, y; + + seq_printf(m, "CRTC %d: pipe: %c, active: %s\n", + crtc->base.base.id, pipe_name(crtc->pipe), + yesno(crtc->active)); + if (crtc->active) { + intel_crtc_info(m, crtc); + + active = cursor_position(dev, crtc->pipe, &x, &y); + seq_printf(m, "\tcursor visible? %s, position (%d, %d), addr 0x%08x, active? %s\n", + yesno(crtc->cursor_base), + x, y, crtc->cursor_addr, + yesno(active)); + } + + seq_printf(m, "\tunderrun reporting: cpu=%s pch=%s \n", + yesno(!crtc->cpu_fifo_underrun_disabled), + yesno(!crtc->pch_fifo_underrun_disabled)); + } + + seq_printf(m, "\n"); + seq_printf(m, "Connector info\n"); + seq_printf(m, "--------------\n"); + list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + intel_connector_info(m, connector); + } + drm_modeset_unlock_all(dev); + intel_runtime_pm_put(dev_priv); + + return 0; +} + struct pipe_crc_info { const char *name; struct drm_device *dev; @@ -2246,7 +2615,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe, *source = INTEL_PIPE_CRC_SOURCE_PIPE; - mutex_lock(&dev->mode_config.mutex); + drm_modeset_lock_all(dev); list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { if (!encoder->base.crtc) @@ -2282,7 +2651,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe, break; } } - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_unlock_all(dev); return ret; } @@ -2332,8 +2701,6 @@ static int vlv_pipe_crc_ctl_reg(struct drm_device *dev, if (need_stable_symbols) { uint32_t tmp = I915_READ(PORT_DFT2_G4X); - WARN_ON(!IS_G4X(dev)); - tmp |= DC_BALANCE_RESET_VLV; if (pipe == PIPE_A) tmp |= PIPE_A_SCRAMBLE_RESET; @@ -2756,11 +3123,179 @@ static const struct file_operations i915_display_crc_ctl_fops = { .write = display_crc_ctl_write }; +static void wm_latency_show(struct seq_file *m, const uint16_t wm[5]) +{ + struct drm_device *dev = m->private; + int num_levels = ilk_wm_max_level(dev) + 1; + int level; + + drm_modeset_lock_all(dev); + + for (level = 0; level < num_levels; level++) { + unsigned int latency = wm[level]; + + /* WM1+ latency values in 0.5us units */ + if (level > 0) + latency *= 5; + + seq_printf(m, "WM%d %u (%u.%u usec)\n", + level, wm[level], + latency / 10, latency % 10); + } + + drm_modeset_unlock_all(dev); +} + +static int pri_wm_latency_show(struct seq_file *m, void *data) +{ + struct drm_device *dev = m->private; + + wm_latency_show(m, to_i915(dev)->wm.pri_latency); + + return 0; +} + +static int spr_wm_latency_show(struct seq_file *m, void *data) +{ + struct drm_device *dev = m->private; + + wm_latency_show(m, to_i915(dev)->wm.spr_latency); + + return 0; +} + +static int cur_wm_latency_show(struct seq_file *m, void *data) +{ + struct drm_device *dev = m->private; + + wm_latency_show(m, to_i915(dev)->wm.cur_latency); + + return 0; +} + +static int pri_wm_latency_open(struct inode *inode, struct file *file) +{ + struct drm_device *dev = inode->i_private; + + if (!HAS_PCH_SPLIT(dev)) + return -ENODEV; + + return single_open(file, pri_wm_latency_show, dev); +} + +static int spr_wm_latency_open(struct inode *inode, struct file *file) +{ + struct drm_device *dev = inode->i_private; + + if (!HAS_PCH_SPLIT(dev)) + return -ENODEV; + + return single_open(file, spr_wm_latency_show, dev); +} + +static int cur_wm_latency_open(struct inode *inode, struct file *file) +{ + struct drm_device *dev = inode->i_private; + + if (!HAS_PCH_SPLIT(dev)) + return -ENODEV; + + return single_open(file, cur_wm_latency_show, dev); +} + +static ssize_t wm_latency_write(struct file *file, const char __user *ubuf, + size_t len, loff_t *offp, uint16_t wm[5]) +{ + struct seq_file *m = file->private_data; + struct drm_device *dev = m->private; + uint16_t new[5] = { 0 }; + int num_levels = ilk_wm_max_level(dev) + 1; + int level; + int ret; + char tmp[32]; + + if (len >= sizeof(tmp)) + return -EINVAL; + + if (copy_from_user(tmp, ubuf, len)) + return -EFAULT; + + tmp[len] = '\0'; + + ret = sscanf(tmp, "%hu %hu %hu %hu %hu", &new[0], &new[1], &new[2], &new[3], &new[4]); + if (ret != num_levels) + return -EINVAL; + + drm_modeset_lock_all(dev); + + for (level = 0; level < num_levels; level++) + wm[level] = new[level]; + + drm_modeset_unlock_all(dev); + + return len; +} + + +static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf, + size_t len, loff_t *offp) +{ + struct seq_file *m = file->private_data; + struct drm_device *dev = m->private; + + return wm_latency_write(file, ubuf, len, offp, to_i915(dev)->wm.pri_latency); +} + +static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf, + size_t len, loff_t *offp) +{ + struct seq_file *m = file->private_data; + struct drm_device *dev = m->private; + + return wm_latency_write(file, ubuf, len, offp, to_i915(dev)->wm.spr_latency); +} + +static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf, + size_t len, loff_t *offp) +{ + struct seq_file *m = file->private_data; + struct drm_device *dev = m->private; + + return wm_latency_write(file, ubuf, len, offp, to_i915(dev)->wm.cur_latency); +} + +static const struct file_operations i915_pri_wm_latency_fops = { + .owner = THIS_MODULE, + .open = pri_wm_latency_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = pri_wm_latency_write +}; + +static const struct file_operations i915_spr_wm_latency_fops = { + .owner = THIS_MODULE, + .open = spr_wm_latency_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = spr_wm_latency_write +}; + +static const struct file_operations i915_cur_wm_latency_fops = { + .owner = THIS_MODULE, + .open = cur_wm_latency_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = cur_wm_latency_write +}; + static int i915_wedged_get(void *data, u64 *val) { struct drm_device *dev = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; *val = atomic_read(&dev_priv->gpu_error.reset_counter); @@ -2771,9 +3306,14 @@ static int i915_wedged_set(void *data, u64 val) { struct drm_device *dev = data; + struct drm_i915_private *dev_priv = dev->dev_private; + + intel_runtime_pm_get(dev_priv); - DRM_INFO("Manually setting wedged to %llu\n", val); - i915_handle_error(dev, val); + i915_handle_error(dev, val, + "Manually setting wedged to %llu", val); + + intel_runtime_pm_put(dev_priv); return 0; } @@ -2786,7 +3326,7 @@ static int i915_ring_stop_get(void *data, u64 *val) { struct drm_device *dev = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; *val = dev_priv->gpu_error.stop_rings; @@ -2929,7 +3469,7 @@ i915_drop_caches_set(void *data, u64 val) list_for_each_entry(vm, &dev_priv->vm_list, global_link) { list_for_each_entry_safe(vma, x, &vm->inactive_list, mm_list) { - if (vma->obj->pin_count) + if (vma->pin_count) continue; ret = i915_vma_unbind(vma); @@ -2963,7 +3503,7 @@ static int i915_max_freq_get(void *data, u64 *val) { struct drm_device *dev = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret; if (!(IS_GEN6(dev) || IS_GEN7(dev))) @@ -2976,9 +3516,9 @@ i915_max_freq_get(void *data, u64 *val) return ret; if (IS_VALLEYVIEW(dev)) - *val = vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay); + *val = vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit); else - *val = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER; + *val = dev_priv->rps.max_freq_softlimit * GT_FREQUENCY_MULTIPLIER; mutex_unlock(&dev_priv->rps.hw_lock); return 0; @@ -2989,6 +3529,7 @@ i915_max_freq_set(void *data, u64 val) { struct drm_device *dev = data; struct drm_i915_private *dev_priv = dev->dev_private; + u32 rp_state_cap, hw_max, hw_min; int ret; if (!(IS_GEN6(dev) || IS_GEN7(dev))) @@ -3007,14 +3548,29 @@ i915_max_freq_set(void *data, u64 val) */ if (IS_VALLEYVIEW(dev)) { val = vlv_freq_opcode(dev_priv, val); - dev_priv->rps.max_delay = val; - valleyview_set_rps(dev, val); + + hw_max = valleyview_rps_max_freq(dev_priv); + hw_min = valleyview_rps_min_freq(dev_priv); } else { do_div(val, GT_FREQUENCY_MULTIPLIER); - dev_priv->rps.max_delay = val; - gen6_set_rps(dev, val); + + rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + hw_max = dev_priv->rps.max_freq; + hw_min = (rp_state_cap >> 16) & 0xff; + } + + if (val < hw_min || val > hw_max || val < dev_priv->rps.min_freq_softlimit) { + mutex_unlock(&dev_priv->rps.hw_lock); + return -EINVAL; } + dev_priv->rps.max_freq_softlimit = val; + + if (IS_VALLEYVIEW(dev)) + valleyview_set_rps(dev, val); + else + gen6_set_rps(dev, val); + mutex_unlock(&dev_priv->rps.hw_lock); return 0; @@ -3028,7 +3584,7 @@ static int i915_min_freq_get(void *data, u64 *val) { struct drm_device *dev = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret; if (!(IS_GEN6(dev) || IS_GEN7(dev))) @@ -3041,9 +3597,9 @@ i915_min_freq_get(void *data, u64 *val) return ret; if (IS_VALLEYVIEW(dev)) - *val = vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay); + *val = vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit); else - *val = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER; + *val = dev_priv->rps.min_freq_softlimit * GT_FREQUENCY_MULTIPLIER; mutex_unlock(&dev_priv->rps.hw_lock); return 0; @@ -3054,6 +3610,7 @@ i915_min_freq_set(void *data, u64 val) { struct drm_device *dev = data; struct drm_i915_private *dev_priv = dev->dev_private; + u32 rp_state_cap, hw_max, hw_min; int ret; if (!(IS_GEN6(dev) || IS_GEN7(dev))) @@ -3072,13 +3629,29 @@ i915_min_freq_set(void *data, u64 val) */ if (IS_VALLEYVIEW(dev)) { val = vlv_freq_opcode(dev_priv, val); - dev_priv->rps.min_delay = val; - valleyview_set_rps(dev, val); + + hw_max = valleyview_rps_max_freq(dev_priv); + hw_min = valleyview_rps_min_freq(dev_priv); } else { do_div(val, GT_FREQUENCY_MULTIPLIER); - dev_priv->rps.min_delay = val; - gen6_set_rps(dev, val); + + rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + hw_max = dev_priv->rps.max_freq; + hw_min = (rp_state_cap >> 16) & 0xff; } + + if (val < hw_min || val > hw_max || val > dev_priv->rps.max_freq_softlimit) { + mutex_unlock(&dev_priv->rps.hw_lock); + return -EINVAL; + } + + dev_priv->rps.min_freq_softlimit = val; + + if (IS_VALLEYVIEW(dev)) + valleyview_set_rps(dev, val); + else + gen6_set_rps(dev, val); + mutex_unlock(&dev_priv->rps.hw_lock); return 0; @@ -3092,7 +3665,7 @@ static int i915_cache_sharing_get(void *data, u64 *val) { struct drm_device *dev = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 snpcr; int ret; @@ -3152,7 +3725,6 @@ static int i915_forcewake_open(struct inode *inode, struct file *file) if (INTEL_INFO(dev)->gen < 6) return 0; - intel_runtime_pm_get(dev_priv); gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); return 0; @@ -3167,7 +3739,6 @@ static int i915_forcewake_release(struct inode *inode, struct file *file) return 0; gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); - intel_runtime_pm_put(dev_priv); return 0; } @@ -3229,7 +3800,7 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS}, {"i915_gem_hws_vebox", i915_hws_info, 0, (void *)VECS}, {"i915_rstdby_delays", i915_rstdby_delays, 0}, - {"i915_cur_delayinfo", i915_cur_delayinfo, 0}, + {"i915_frequency_info", i915_frequency_info, 0}, {"i915_delayfreq_table", i915_delayfreq_table, 0}, {"i915_inttoext_table", i915_inttoext_table, 0}, {"i915_drpc_info", i915_drpc_info, 0}, @@ -3245,12 +3816,13 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_gen6_forcewake_count", i915_gen6_forcewake_count_info, 0}, {"i915_swizzle_info", i915_swizzle_info, 0}, {"i915_ppgtt_info", i915_ppgtt_info, 0}, - {"i915_dpio", i915_dpio_info, 0}, {"i915_llc", i915_llc, 0}, {"i915_edp_psr_status", i915_edp_psr_status, 0}, + {"i915_sink_crc_eDP1", i915_sink_crc, 0}, {"i915_energy_uJ", i915_energy_uJ, 0}, {"i915_pc8_status", i915_pc8_status, 0}, {"i915_power_domain_info", i915_power_domain_info, 0}, + {"i915_display_info", i915_display_info, 0}, }; #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list) @@ -3269,6 +3841,9 @@ static const struct i915_debugfs_files { {"i915_error_state", &i915_error_state_fops}, {"i915_next_seqno", &i915_next_seqno_fops}, {"i915_display_crc_ctl", &i915_display_crc_ctl_fops}, + {"i915_pri_wm_latency", &i915_pri_wm_latency_fops}, + {"i915_spr_wm_latency", &i915_spr_wm_latency_fops}, + {"i915_cur_wm_latency", &i915_cur_wm_latency_fops}, }; void intel_display_crc_init(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c index 15a74f979b4..d4434414062 100644 --- a/drivers/gpu/drm/i915/i915_dma.c +++ b/drivers/gpu/drm/i915/i915_dma.c @@ -36,6 +36,8 @@ #include "i915_drv.h" #include "i915_trace.h" #include <linux/pci.h> +#include <linux/console.h> +#include <linux/vt.h> #include <linux/vgaarb.h> #include <linux/acpi.h> #include <linux/pnp.h> @@ -44,6 +46,7 @@ #include <acpi/video.h> #include <linux/pm.h> #include <linux/pm_runtime.h> +#include <linux/oom.h> #define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS]) @@ -63,7 +66,7 @@ * has access to the ring. */ #define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \ - if (LP_RING(dev->dev_private)->obj == NULL) \ + if (LP_RING(dev->dev_private)->buffer->obj == NULL) \ LOCK_TEST_WITH_RETURN(dev, file); \ } while (0) @@ -82,7 +85,7 @@ intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg) void i915_update_dri1_breadcrumb(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv; /* @@ -103,7 +106,7 @@ void i915_update_dri1_breadcrumb(struct drm_device *dev) static void i915_write_hws_pga(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 addr; addr = dev_priv->status_page_dmah->busaddr; @@ -118,8 +121,8 @@ static void i915_write_hws_pga(struct drm_device *dev) */ static void i915_free_hws(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = LP_RING(dev_priv); + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = LP_RING(dev_priv); if (dev_priv->status_page_dmah) { drm_pci_free(dev, dev_priv->status_page_dmah); @@ -137,9 +140,10 @@ static void i915_free_hws(struct drm_device *dev) void i915_kernel_lost_context(struct drm_device * dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv; - struct intel_ring_buffer *ring = LP_RING(dev_priv); + struct intel_engine_cs *ring = LP_RING(dev_priv); + struct intel_ringbuffer *ringbuf = ring->buffer; /* * We should never lose context on the ring with modesetting @@ -148,23 +152,23 @@ void i915_kernel_lost_context(struct drm_device * dev) if (drm_core_check_feature(dev, DRIVER_MODESET)) return; - ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; - ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; - ring->space = ring->head - (ring->tail + I915_RING_FREE_SPACE); - if (ring->space < 0) - ring->space += ring->size; + ringbuf->head = I915_READ_HEAD(ring) & HEAD_ADDR; + ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; + ringbuf->space = ringbuf->head - (ringbuf->tail + I915_RING_FREE_SPACE); + if (ringbuf->space < 0) + ringbuf->space += ringbuf->size; if (!dev->primary->master) return; master_priv = dev->primary->master->driver_priv; - if (ring->head == ring->tail && master_priv->sarea_priv) + if (ringbuf->head == ringbuf->tail && master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY; } static int i915_dma_cleanup(struct drm_device * dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int i; /* Make sure interrupts are disabled here because the uninstall ioctl @@ -188,7 +192,7 @@ static int i915_dma_cleanup(struct drm_device * dev) static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret; @@ -201,7 +205,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) } if (init->ring_size != 0) { - if (LP_RING(dev_priv)->obj != NULL) { + if (LP_RING(dev_priv)->buffer->obj != NULL) { i915_dma_cleanup(dev); DRM_ERROR("Client tried to initialize ringbuffer in " "GEM mode\n"); @@ -233,12 +237,12 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) static int i915_dma_resume(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - struct intel_ring_buffer *ring = LP_RING(dev_priv); + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = LP_RING(dev_priv); DRM_DEBUG_DRIVER("%s\n", __func__); - if (ring->virtual_start == NULL) { + if (ring->buffer->virtual_start == NULL) { DRM_ERROR("can not ioremap virtual address for" " ring buffer\n"); return -ENOMEM; @@ -357,10 +361,10 @@ static int validate_cmd(int cmd) static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int i, ret; - if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8) + if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->buffer->size - 8) return -EINVAL; for (i = 0; i < dwords;) { @@ -431,7 +435,7 @@ i915_emit_box(struct drm_device *dev, static void i915_emit_breadcrumb(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; dev_priv->dri1.counter++; @@ -547,7 +551,7 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev, static int i915_dispatch_flip(struct drm_device * dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret; @@ -625,10 +629,9 @@ static int i915_flush_ioctl(struct drm_device *dev, void *data, static int i915_batchbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) - master_priv->sarea_priv; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_master_private *master_priv; + drm_i915_sarea_t *sarea_priv; drm_i915_batchbuffer_t *batch = data; int ret; struct drm_clip_rect *cliprects = NULL; @@ -636,6 +639,9 @@ static int i915_batchbuffer(struct drm_device *dev, void *data, if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; + master_priv = dev->primary->master->driver_priv; + sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv; + if (!dev_priv->dri1.allow_batchbuffer) { DRM_ERROR("Batchbuffer ioctl disabled\n"); return -EINVAL; @@ -681,10 +687,9 @@ fail_free: static int i915_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; - drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) - master_priv->sarea_priv; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_master_private *master_priv; + drm_i915_sarea_t *sarea_priv; drm_i915_cmdbuffer_t *cmdbuf = data; struct drm_clip_rect *cliprects = NULL; void *batch_data; @@ -696,6 +701,9 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data, if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; + master_priv = dev->primary->master->driver_priv; + sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv; + RING_LOCK_TEST_WITH_RETURN(dev, file_priv); if (cmdbuf->num_cliprects < 0) @@ -749,7 +757,7 @@ fail_batch_free: static int i915_emit_irq(struct drm_device * dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; i915_kernel_lost_context(dev); @@ -775,10 +783,10 @@ static int i915_emit_irq(struct drm_device * dev) static int i915_wait_irq(struct drm_device * dev, int irq_nr) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret = 0; - struct intel_ring_buffer *ring = LP_RING(dev_priv); + struct intel_engine_cs *ring = LP_RING(dev_priv); DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv)); @@ -812,14 +820,14 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr) static int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_irq_emit_t *emit = data; int result; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; - if (!dev_priv || !LP_RING(dev_priv)->virtual_start) { + if (!dev_priv || !LP_RING(dev_priv)->buffer->virtual_start) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } @@ -843,7 +851,7 @@ static int i915_irq_emit(struct drm_device *dev, void *data, static int i915_irq_wait(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_irq_wait_t *irqwait = data; if (drm_core_check_feature(dev, DRIVER_MODESET)) @@ -860,7 +868,7 @@ static int i915_irq_wait(struct drm_device *dev, void *data, static int i915_vblank_pipe_get(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_vblank_pipe_t *pipe = data; if (drm_core_check_feature(dev, DRIVER_MODESET)) @@ -921,7 +929,7 @@ static int i915_flip_bufs(struct drm_device *dev, void *data, static int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_getparam_t *param = data; int value; @@ -990,7 +998,7 @@ static int i915_getparam(struct drm_device *dev, void *data, value = HAS_WT(dev); break; case I915_PARAM_HAS_ALIASING_PPGTT: - value = dev_priv->mm.aliasing_ppgtt ? 1 : 0; + value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev); break; case I915_PARAM_HAS_WAIT_TIMEOUT: value = 1; @@ -1013,6 +1021,9 @@ static int i915_getparam(struct drm_device *dev, void *data, case I915_PARAM_HAS_EXEC_HANDLE_LUT: value = 1; break; + case I915_PARAM_CMD_PARSER_VERSION: + value = i915_cmd_parser_get_version(); + break; default: DRM_DEBUG("Unknown parameter %d\n", param->param); return -EINVAL; @@ -1029,7 +1040,7 @@ static int i915_getparam(struct drm_device *dev, void *data, static int i915_setparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_setparam_t *param = data; if (!dev_priv) { @@ -1064,9 +1075,9 @@ static int i915_setparam(struct drm_device *dev, void *data, static int i915_set_status_page(struct drm_device *dev, void *data, struct drm_file *file_priv) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_hws_addr_t *hws = data; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; @@ -1132,7 +1143,7 @@ static int i915_get_bridge_dev(struct drm_device *dev) static int intel_alloc_mchbar_resource(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp_lo, temp_hi = 0; u64 mchbar_addr; @@ -1178,11 +1189,14 @@ intel_alloc_mchbar_resource(struct drm_device *dev) static void intel_setup_mchbar(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp; bool enabled; + if (IS_VALLEYVIEW(dev)) + return; + dev_priv->mchbar_need_disable = false; if (IS_I915G(dev) || IS_I915GM(dev)) { @@ -1215,7 +1229,7 @@ intel_setup_mchbar(struct drm_device *dev) static void intel_teardown_mchbar(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; u32 temp; @@ -1270,12 +1284,13 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_ static bool i915_switcheroo_can_switch(struct pci_dev *pdev) { struct drm_device *dev = pci_get_drvdata(pdev); - bool can_switch; - spin_lock(&dev->count_lock); - can_switch = (dev->open_count == 0); - spin_unlock(&dev->count_lock); - return can_switch; + /* + * FIXME: open_count is protected by drm_global_mutex but that would lead to + * locking inversion with the driver load path. And the access here is + * completely racy anyway. So don't bother with locking for now. + */ + return dev->open_count == 0; } static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { @@ -1317,19 +1332,19 @@ static int i915_load_modeset_init(struct drm_device *dev) if (ret) goto cleanup_vga_switcheroo; - ret = drm_irq_install(dev); + intel_power_domains_init_hw(dev_priv); + + ret = drm_irq_install(dev, dev->pdev->irq); if (ret) goto cleanup_gem_stolen; - intel_power_domains_init_hw(dev); - /* Important: The output setup functions called by modeset_init need * working irqs for e.g. gmbus and dp aux transfers. */ intel_modeset_init(dev); ret = i915_gem_init(dev); if (ret) - goto cleanup_power; + goto cleanup_irq; INIT_WORK(&dev_priv->console_resume_work, intel_console_resume); @@ -1338,10 +1353,8 @@ static int i915_load_modeset_init(struct drm_device *dev) /* Always safe in the mode setting case. */ /* FIXME: do pre/post-mode set stuff in core KMS code */ dev->vblank_disable_allowed = true; - if (INTEL_INFO(dev)->num_pipes == 0) { - intel_display_power_put(dev, POWER_DOMAIN_VGA); + if (INTEL_INFO(dev)->num_pipes == 0) return 0; - } ret = intel_fbdev_init(dev); if (ret) @@ -1374,10 +1387,8 @@ cleanup_gem: i915_gem_cleanup_ringbuffer(dev); i915_gem_context_fini(dev); mutex_unlock(&dev->struct_mutex); - i915_gem_cleanup_aliasing_ppgtt(dev); - drm_mm_takedown(&dev_priv->gtt.base.mm); -cleanup_power: - intel_display_power_put(dev, POWER_DOMAIN_VGA); + WARN_ON(dev_priv->mm.aliasing_ppgtt); +cleanup_irq: drm_irq_uninstall(dev); cleanup_gem_stolen: i915_gem_cleanup_stolen(dev); @@ -1440,9 +1451,42 @@ static void i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) } #endif +#if !defined(CONFIG_VGA_CONSOLE) +static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) +{ + return 0; +} +#elif !defined(CONFIG_DUMMY_CONSOLE) +static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) +{ + return -ENODEV; +} +#else +static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) +{ + int ret = 0; + + DRM_INFO("Replacing VGA console driver\n"); + + console_lock(); + if (con_is_bound(&vga_con)) + ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); + if (ret == 0) { + ret = do_unregister_con_driver(&vga_con); + + /* Ignore "already unregistered". */ + if (ret == -ENODEV) + ret = 0; + } + console_unlock(); + + return ret; +} +#endif + static void i915_dump_device_info(struct drm_i915_private *dev_priv) { - const struct intel_device_info *info = dev_priv->info; + const struct intel_device_info *info = &dev_priv->info; #define PRINT_S(name) "%s" #define SEP_EMPTY @@ -1459,6 +1503,62 @@ static void i915_dump_device_info(struct drm_i915_private *dev_priv) #undef SEP_COMMA } +/* + * Determine various intel_device_info fields at runtime. + * + * Use it when either: + * - it's judged too laborious to fill n static structures with the limit + * when a simple if statement does the job, + * - run-time checks (eg read fuse/strap registers) are needed. + * + * This function needs to be called: + * - after the MMIO has been setup as we are reading registers, + * - after the PCH has been detected, + * - before the first usage of the fields it can tweak. + */ +static void intel_device_info_runtime_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_device_info *info; + enum pipe pipe; + + info = (struct intel_device_info *)&dev_priv->info; + + if (IS_VALLEYVIEW(dev)) + for_each_pipe(pipe) + info->num_sprites[pipe] = 2; + else + for_each_pipe(pipe) + info->num_sprites[pipe] = 1; + + if (i915.disable_display) { + DRM_INFO("Display disabled (module parameter)\n"); + info->num_pipes = 0; + } else if (info->num_pipes > 0 && + (INTEL_INFO(dev)->gen == 7 || INTEL_INFO(dev)->gen == 8) && + !IS_VALLEYVIEW(dev)) { + u32 fuse_strap = I915_READ(FUSE_STRAP); + u32 sfuse_strap = I915_READ(SFUSE_STRAP); + + /* + * SFUSE_STRAP is supposed to have a bit signalling the display + * is fused off. Unfortunately it seems that, at least in + * certain cases, fused off display means that PCH display + * reads don't land anywhere. In that case, we read 0s. + * + * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK + * should be set when taking over after the firmware. + */ + if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE || + sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED || + (dev_priv->pch_type == PCH_CPT && + !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) { + DRM_INFO("Display fused off, disabling\n"); + info->num_pipes = 0; + } + } +} + /** * i915_driver_load - setup chip and create an initial config * @dev: DRM device @@ -1473,7 +1573,7 @@ static void i915_dump_device_info(struct drm_i915_private *dev_priv) int i915_driver_load(struct drm_device *dev, unsigned long flags) { struct drm_i915_private *dev_priv; - struct intel_device_info *info; + struct intel_device_info *info, *device_info; int ret = 0, mmio_bar, mmio_size; uint32_t aperture_size; @@ -1496,7 +1596,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) dev->dev_private = (void *)dev_priv; dev_priv->dev = dev; - dev_priv->info = info; + + /* copy initial configuration to dev_priv->info */ + device_info = (struct intel_device_info *)&dev_priv->info; + *device_info = *info; spin_lock_init(&dev_priv->irq_lock); spin_lock_init(&dev_priv->gpu_error.lock); @@ -1545,8 +1648,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) goto put_bridge; } - intel_uncore_early_sanitize(dev); - /* This must be called before any calls to HAS_PCH_* */ intel_detect_pch(dev); @@ -1556,8 +1657,15 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) if (ret) goto out_regs; - if (drm_core_check_feature(dev, DRIVER_MODESET)) + if (drm_core_check_feature(dev, DRIVER_MODESET)) { + ret = i915_kick_out_vgacon(dev_priv); + if (ret) { + DRM_ERROR("failed to remove conflicting VGA console\n"); + goto out_gtt; + } + i915_kick_out_firmware_fb(dev_priv); + } pci_set_master(dev->pdev); @@ -1635,9 +1743,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) if (!IS_I945G(dev) && !IS_I945GM(dev)) pci_enable_msi(dev->pdev); - dev_priv->num_plane = 1; - if (IS_VALLEYVIEW(dev)) - dev_priv->num_plane = 2; + intel_device_info_runtime_init(dev); if (INTEL_INFO(dev)->num_pipes) { ret = drm_vblank_init(dev, INTEL_INFO(dev)->num_pipes); @@ -1645,7 +1751,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) goto out_gem_unload; } - intel_power_domains_init(dev); + intel_power_domains_init(dev_priv); if (drm_core_check_feature(dev, DRIVER_MODESET)) { ret = i915_load_modeset_init(dev); @@ -1674,11 +1780,11 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) return 0; out_power_well: - intel_power_domains_remove(dev); + intel_power_domains_remove(dev_priv); drm_vblank_cleanup(dev); out_gem_unload: - if (dev_priv->mm.inactive_shrinker.scan_objects) - unregister_shrinker(&dev_priv->mm.inactive_shrinker); + WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier)); + unregister_shrinker(&dev_priv->mm.shrinker); if (dev->pdev->msi_enabled) pci_disable_msi(dev->pdev); @@ -1691,8 +1797,6 @@ out_mtrrfree: arch_phys_wc_del(dev_priv->gtt.mtrr); io_mapping_free(dev_priv->gtt.mappable); out_gtt: - list_del(&dev_priv->gtt.base.global_link); - drm_mm_takedown(&dev_priv->gtt.base.mm); dev_priv->gtt.base.cleanup(&dev_priv->gtt.base); out_regs: intel_uncore_fini(dev); @@ -1724,13 +1828,13 @@ int i915_driver_unload(struct drm_device *dev) /* The i915.ko module is still not prepared to be loaded when * the power well is not enabled, so just enable it in case * we're going to unload/reload. */ - intel_display_set_init_power(dev, true); - intel_power_domains_remove(dev); + intel_display_set_init_power(dev_priv, true); + intel_power_domains_remove(dev_priv); i915_teardown_sysfs(dev); - if (dev_priv->mm.inactive_shrinker.scan_objects) - unregister_shrinker(&dev_priv->mm.inactive_shrinker); + WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier)); + unregister_shrinker(&dev_priv->mm.shrinker); io_mapping_free(dev_priv->gtt.mappable); arch_phys_wc_del(dev_priv->gtt.mtrr); @@ -1761,8 +1865,6 @@ int i915_driver_unload(struct drm_device *dev) cancel_work_sync(&dev_priv->gpu_error.work); i915_destroy_error_state(dev); - cancel_delayed_work_sync(&dev_priv->pc8.enable_work); - if (dev->pdev->msi_enabled) pci_disable_msi(dev->pdev); @@ -1773,18 +1875,16 @@ int i915_driver_unload(struct drm_device *dev) flush_workqueue(dev_priv->wq); mutex_lock(&dev->struct_mutex); - i915_gem_free_all_phys_object(dev); i915_gem_cleanup_ringbuffer(dev); i915_gem_context_fini(dev); + WARN_ON(dev_priv->mm.aliasing_ppgtt); mutex_unlock(&dev->struct_mutex); - i915_gem_cleanup_aliasing_ppgtt(dev); i915_gem_cleanup_stolen(dev); if (!I915_NEED_GFX_HWS(dev)) i915_free_hws(dev); } - list_del(&dev_priv->gtt.base.global_link); WARN_ON(!list_empty(&dev_priv->vm_list)); drm_vblank_cleanup(dev); @@ -1805,7 +1905,7 @@ int i915_driver_unload(struct drm_device *dev) kmem_cache_destroy(dev_priv->slab); pci_dev_put(dev_priv->bridge_dev); - kfree(dev->dev_private); + kfree(dev_priv); return 0; } @@ -1835,7 +1935,7 @@ int i915_driver_open(struct drm_device *dev, struct drm_file *file) */ void i915_driver_lastclose(struct drm_device * dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; /* On gen6+ we refuse to init without kms enabled, but then the drm core * goes right around and calls lastclose. Check for this and don't clean @@ -1866,6 +1966,8 @@ void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; + if (file_priv && file_priv->bsd_ring) + file_priv->bsd_ring = NULL; kfree(file_priv); } @@ -1919,9 +2021,10 @@ const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), }; -int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls); +int i915_max_ioctl = ARRAY_SIZE(i915_ioctls); /* * This is really ugly: Because old userspace abused the linux agp interface to diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 04f1f02c401..651e65e051c 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -36,136 +36,52 @@ #include <linux/console.h> #include <linux/module.h> +#include <linux/pm_runtime.h> #include <drm/drm_crtc_helper.h> -static int i915_modeset __read_mostly = -1; -module_param_named(modeset, i915_modeset, int, 0400); -MODULE_PARM_DESC(modeset, - "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, " - "1=on, -1=force vga console preference [default])"); - -unsigned int i915_fbpercrtc __always_unused = 0; -module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400); - -int i915_panel_ignore_lid __read_mostly = 1; -module_param_named(panel_ignore_lid, i915_panel_ignore_lid, int, 0600); -MODULE_PARM_DESC(panel_ignore_lid, - "Override lid status (0=autodetect, 1=autodetect disabled [default], " - "-1=force lid closed, -2=force lid open)"); - -unsigned int i915_powersave __read_mostly = 1; -module_param_named(powersave, i915_powersave, int, 0600); -MODULE_PARM_DESC(powersave, - "Enable powersavings, fbc, downclocking, etc. (default: true)"); - -int i915_semaphores __read_mostly = -1; -module_param_named(semaphores, i915_semaphores, int, 0400); -MODULE_PARM_DESC(semaphores, - "Use semaphores for inter-ring sync (default: -1 (use per-chip defaults))"); - -int i915_enable_rc6 __read_mostly = -1; -module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0400); -MODULE_PARM_DESC(i915_enable_rc6, - "Enable power-saving render C-state 6. " - "Different stages can be selected via bitmask values " - "(0 = disable; 1 = enable rc6; 2 = enable deep rc6; 4 = enable deepest rc6). " - "For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. " - "default: -1 (use per-chip default)"); - -int i915_enable_fbc __read_mostly = -1; -module_param_named(i915_enable_fbc, i915_enable_fbc, int, 0600); -MODULE_PARM_DESC(i915_enable_fbc, - "Enable frame buffer compression for power savings " - "(default: -1 (use per-chip default))"); - -unsigned int i915_lvds_downclock __read_mostly = 0; -module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400); -MODULE_PARM_DESC(lvds_downclock, - "Use panel (LVDS/eDP) downclocking for power savings " - "(default: false)"); - -int i915_lvds_channel_mode __read_mostly; -module_param_named(lvds_channel_mode, i915_lvds_channel_mode, int, 0600); -MODULE_PARM_DESC(lvds_channel_mode, - "Specify LVDS channel mode " - "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)"); - -int i915_panel_use_ssc __read_mostly = -1; -module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600); -MODULE_PARM_DESC(lvds_use_ssc, - "Use Spread Spectrum Clock with panels [LVDS/eDP] " - "(default: auto from VBT)"); - -int i915_vbt_sdvo_panel_type __read_mostly = -1; -module_param_named(vbt_sdvo_panel_type, i915_vbt_sdvo_panel_type, int, 0600); -MODULE_PARM_DESC(vbt_sdvo_panel_type, - "Override/Ignore selection of SDVO panel mode in the VBT " - "(-2=ignore, -1=auto [default], index in VBT BIOS table)"); - -static bool i915_try_reset __read_mostly = true; -module_param_named(reset, i915_try_reset, bool, 0600); -MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)"); - -bool i915_enable_hangcheck __read_mostly = true; -module_param_named(enable_hangcheck, i915_enable_hangcheck, bool, 0644); -MODULE_PARM_DESC(enable_hangcheck, - "Periodically check GPU activity for detecting hangs. " - "WARNING: Disabling this can cause system wide hangs. " - "(default: true)"); - -int i915_enable_ppgtt __read_mostly = -1; -module_param_named(i915_enable_ppgtt, i915_enable_ppgtt, int, 0400); -MODULE_PARM_DESC(i915_enable_ppgtt, - "Enable PPGTT (default: true)"); - -int i915_enable_psr __read_mostly = 0; -module_param_named(enable_psr, i915_enable_psr, int, 0600); -MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)"); - -unsigned int i915_preliminary_hw_support __read_mostly = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT); -module_param_named(preliminary_hw_support, i915_preliminary_hw_support, int, 0600); -MODULE_PARM_DESC(preliminary_hw_support, - "Enable preliminary hardware support."); - -int i915_disable_power_well __read_mostly = 1; -module_param_named(disable_power_well, i915_disable_power_well, int, 0600); -MODULE_PARM_DESC(disable_power_well, - "Disable the power well when possible (default: true)"); - -int i915_enable_ips __read_mostly = 1; -module_param_named(enable_ips, i915_enable_ips, int, 0600); -MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)"); - -bool i915_fastboot __read_mostly = 0; -module_param_named(fastboot, i915_fastboot, bool, 0600); -MODULE_PARM_DESC(fastboot, "Try to skip unnecessary mode sets at boot time " - "(default: false)"); - -int i915_enable_pc8 __read_mostly = 1; -module_param_named(enable_pc8, i915_enable_pc8, int, 0600); -MODULE_PARM_DESC(enable_pc8, "Enable support for low power package C states (PC8+) (default: true)"); - -int i915_pc8_timeout __read_mostly = 5000; -module_param_named(pc8_timeout, i915_pc8_timeout, int, 0600); -MODULE_PARM_DESC(pc8_timeout, "Number of msecs of idleness required to enter PC8+ (default: 5000)"); - -bool i915_prefault_disable __read_mostly; -module_param_named(prefault_disable, i915_prefault_disable, bool, 0600); -MODULE_PARM_DESC(prefault_disable, - "Disable page prefaulting for pread/pwrite/reloc (default:false). For developers only."); - static struct drm_driver driver; +#define GEN_DEFAULT_PIPEOFFSETS \ + .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ + PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \ + .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ + TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \ + .dpll_offsets = { DPLL_A_OFFSET, DPLL_B_OFFSET }, \ + .dpll_md_offsets = { DPLL_A_MD_OFFSET, DPLL_B_MD_OFFSET }, \ + .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET } + +#define GEN_CHV_PIPEOFFSETS \ + .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ + CHV_PIPE_C_OFFSET }, \ + .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ + CHV_TRANSCODER_C_OFFSET, }, \ + .dpll_offsets = { DPLL_A_OFFSET, DPLL_B_OFFSET, \ + CHV_DPLL_C_OFFSET }, \ + .dpll_md_offsets = { DPLL_A_MD_OFFSET, DPLL_B_MD_OFFSET, \ + CHV_DPLL_C_MD_OFFSET }, \ + .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \ + CHV_PALETTE_C_OFFSET } + +#define CURSOR_OFFSETS \ + .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET } + +#define IVB_CURSOR_OFFSETS \ + .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET } + static const struct intel_device_info intel_i830_info = { .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_845g_info = { .gen = 2, .num_pipes = 1, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i85x_info = { @@ -174,18 +90,24 @@ static const struct intel_device_info intel_i85x_info = { .has_overlay = 1, .overlay_needs_physical = 1, .has_fbc = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i865g_info = { .gen = 2, .num_pipes = 1, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i915g_info = { .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i915gm_info = { .gen = 3, .is_mobile = 1, .num_pipes = 2, @@ -194,11 +116,15 @@ static const struct intel_device_info intel_i915gm_info = { .supports_tv = 1, .has_fbc = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i945g_info = { .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2, .has_overlay = 1, .overlay_needs_physical = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i945gm_info = { .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2, @@ -207,6 +133,8 @@ static const struct intel_device_info intel_i945gm_info = { .supports_tv = 1, .has_fbc = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i965g_info = { @@ -214,6 +142,8 @@ static const struct intel_device_info intel_i965g_info = { .has_hotplug = 1, .has_overlay = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_i965gm_info = { @@ -222,6 +152,8 @@ static const struct intel_device_info intel_i965gm_info = { .has_overlay = 1, .supports_tv = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_g33_info = { @@ -229,12 +161,16 @@ static const struct intel_device_info intel_g33_info = { .need_gfx_hws = 1, .has_hotplug = 1, .has_overlay = 1, .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_g45_info = { .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2, .has_pipe_cxsr = 1, .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_gm45_info = { @@ -243,18 +179,24 @@ static const struct intel_device_info intel_gm45_info = { .has_pipe_cxsr = 1, .has_hotplug = 1, .supports_tv = 1, .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_pineview_info = { .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2, .need_gfx_hws = 1, .has_hotplug = 1, .has_overlay = 1, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_ironlake_d_info = { .gen = 5, .num_pipes = 2, .need_gfx_hws = 1, .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_ironlake_m_info = { @@ -262,6 +204,8 @@ static const struct intel_device_info intel_ironlake_m_info = { .need_gfx_hws = 1, .has_hotplug = 1, .has_fbc = 1, .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_sandybridge_d_info = { @@ -270,6 +214,8 @@ static const struct intel_device_info intel_sandybridge_d_info = { .has_fbc = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING, .has_llc = 1, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_sandybridge_m_info = { @@ -278,6 +224,8 @@ static const struct intel_device_info intel_sandybridge_m_info = { .has_fbc = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING, .has_llc = 1, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; #define GEN7_FEATURES \ @@ -290,18 +238,24 @@ static const struct intel_device_info intel_sandybridge_m_info = { static const struct intel_device_info intel_ivybridge_d_info = { GEN7_FEATURES, .is_ivybridge = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_ivybridge_m_info = { GEN7_FEATURES, .is_ivybridge = 1, .is_mobile = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_ivybridge_q_info = { GEN7_FEATURES, .is_ivybridge = 1, .num_pipes = 0, /* legal, last one wins */ + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_valleyview_m_info = { @@ -312,6 +266,8 @@ static const struct intel_device_info intel_valleyview_m_info = { .display_mmio_offset = VLV_DISPLAY_BASE, .has_fbc = 0, /* legal, last one wins */ .has_llc = 0, /* legal, last one wins */ + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_valleyview_d_info = { @@ -321,6 +277,8 @@ static const struct intel_device_info intel_valleyview_d_info = { .display_mmio_offset = VLV_DISPLAY_BASE, .has_fbc = 0, /* legal, last one wins */ .has_llc = 0, /* legal, last one wins */ + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, }; static const struct intel_device_info intel_haswell_d_info = { @@ -329,6 +287,8 @@ static const struct intel_device_info intel_haswell_d_info = { .has_ddi = 1, .has_fpga_dbg = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_haswell_m_info = { @@ -338,6 +298,8 @@ static const struct intel_device_info intel_haswell_m_info = { .has_ddi = 1, .has_fpga_dbg = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broadwell_d_info = { @@ -346,6 +308,9 @@ static const struct intel_device_info intel_broadwell_d_info = { .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .has_llc = 1, .has_ddi = 1, + .has_fbc = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, }; static const struct intel_device_info intel_broadwell_m_info = { @@ -354,6 +319,42 @@ static const struct intel_device_info intel_broadwell_m_info = { .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .has_llc = 1, .has_ddi = 1, + .has_fbc = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_broadwell_gt3d_info = { + .gen = 8, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, + .has_llc = 1, + .has_ddi = 1, + .has_fbc = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_broadwell_gt3m_info = { + .gen = 8, .is_mobile = 1, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, + .has_llc = 1, + .has_ddi = 1, + .has_fbc = 1, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_cherryview_info = { + .is_preliminary = 1, + .gen = 8, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, + .is_valleyview = 1, + .display_mmio_offset = VLV_DISPLAY_BASE, + GEN_CHV_PIPEOFFSETS, + CURSOR_OFFSETS, }; /* @@ -388,8 +389,11 @@ static const struct intel_device_info intel_broadwell_m_info = { INTEL_HSW_M_IDS(&intel_haswell_m_info), \ INTEL_VLV_M_IDS(&intel_valleyview_m_info), \ INTEL_VLV_D_IDS(&intel_valleyview_d_info), \ - INTEL_BDW_M_IDS(&intel_broadwell_m_info), \ - INTEL_BDW_D_IDS(&intel_broadwell_d_info) + INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info), \ + INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info), \ + INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), \ + INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), \ + INTEL_CHV_IDS(&intel_cherryview_info) static const struct pci_device_id pciidlist[] = { /* aka */ INTEL_PCI_IDS, @@ -403,7 +407,7 @@ MODULE_DEVICE_TABLE(pci, pciidlist); void intel_detect_pch(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct pci_dev *pch; + struct pci_dev *pch = NULL; /* In all current cases, num_pipes is equivalent to the PCH_NOP setting * (which really amounts to a PCH but no South Display). @@ -424,12 +428,9 @@ void intel_detect_pch(struct drm_device *dev) * all the ISA bridge devices and check for the first match, instead * of only checking the first one. */ - pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL); - while (pch) { - struct pci_dev *curr = pch; + while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) { if (pch->vendor == PCI_VENDOR_ID_INTEL) { - unsigned short id; - id = pch->device & INTEL_PCH_DEVICE_ID_MASK; + unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK; dev_priv->pch_id = id; if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) { @@ -461,18 +462,16 @@ void intel_detect_pch(struct drm_device *dev) DRM_DEBUG_KMS("Found LynxPoint LP PCH\n"); WARN_ON(!IS_HASWELL(dev)); WARN_ON(!IS_ULT(dev)); - } else { - goto check_next; - } - pci_dev_put(pch); + } else + continue; + break; } -check_next: - pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, curr); - pci_dev_put(curr); } if (!pch) - DRM_DEBUG_KMS("No PCH found?\n"); + DRM_DEBUG_KMS("No PCH found.\n"); + + pci_dev_put(pch); } bool i915_semaphore_is_enabled(struct drm_device *dev) @@ -480,14 +479,12 @@ bool i915_semaphore_is_enabled(struct drm_device *dev) if (INTEL_INFO(dev)->gen < 6) return false; + if (i915.semaphores >= 0) + return i915.semaphores; + /* Until we get further testing... */ - if (IS_GEN8(dev)) { - WARN_ON(!i915_preliminary_hw_support); + if (IS_GEN8(dev)) return false; - } - - if (i915_semaphores >= 0) - return i915_semaphores; #ifdef CONFIG_INTEL_IOMMU /* Enable semaphores on SNB when IO remapping is off */ @@ -512,8 +509,7 @@ static int i915_drm_freeze(struct drm_device *dev) /* We do a lot of poking in a lot of registers, make sure they work * properly. */ - hsw_disable_package_c8(dev_priv); - intel_display_set_init_power(dev, true); + intel_display_set_init_power(dev_priv, true); drm_kms_helper_poll_disable(dev); @@ -530,18 +526,20 @@ static int i915_drm_freeze(struct drm_device *dev) return error; } - cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work); - drm_irq_uninstall(dev); dev_priv->enable_hotplug_processing = false; + + intel_disable_gt_powersave(dev); + /* * Disable CRTCs directly since we want to preserve sw state * for _thaw. */ - mutex_lock(&dev->mode_config.mutex); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + drm_modeset_lock_all(dev); + for_each_crtc(dev, crtc) { dev_priv->display.crtc_disable(crtc); - mutex_unlock(&dev->mode_config.mutex); + } + drm_modeset_unlock_all(dev); intel_modeset_suspend_hw(dev); } @@ -551,11 +549,14 @@ static int i915_drm_freeze(struct drm_device *dev) i915_save_state(dev); intel_opregion_fini(dev); + intel_uncore_fini(dev); console_lock(); intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED); console_unlock(); + dev_priv->suspend_count++; + return 0; } @@ -601,32 +602,20 @@ void intel_console_resume(struct work_struct *work) console_unlock(); } -static void intel_resume_hotplug(struct drm_device *dev) +static int i915_drm_thaw_early(struct drm_device *dev) { - struct drm_mode_config *mode_config = &dev->mode_config; - struct intel_encoder *encoder; - - mutex_lock(&mode_config->mutex); - DRM_DEBUG_KMS("running encoder hotplug functions\n"); - - list_for_each_entry(encoder, &mode_config->encoder_list, base.head) - if (encoder->hot_plug) - encoder->hot_plug(encoder); + struct drm_i915_private *dev_priv = dev->dev_private; - mutex_unlock(&mode_config->mutex); + intel_uncore_early_sanitize(dev); + intel_uncore_sanitize(dev); + intel_power_domains_init_hw(dev_priv); - /* Just fire off a uevent and let userspace tell us what to do */ - drm_helper_hpd_irq_event(dev); + return 0; } static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) { struct drm_i915_private *dev_priv = dev->dev_private; - int error = 0; - - intel_uncore_early_sanitize(dev); - - intel_uncore_sanitize(dev); if (drm_core_check_feature(dev, DRIVER_MODESET) && restore_gtt_mappings) { @@ -635,27 +624,27 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) mutex_unlock(&dev->struct_mutex); } - intel_power_domains_init_hw(dev); - i915_restore_state(dev); intel_opregion_setup(dev); /* KMS EnterVT equivalent */ if (drm_core_check_feature(dev, DRIVER_MODESET)) { intel_init_pch_refclk(dev); + drm_mode_config_reset(dev); mutex_lock(&dev->struct_mutex); - - error = i915_gem_init_hw(dev); + if (i915_gem_init_hw(dev)) { + DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); + atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + } mutex_unlock(&dev->struct_mutex); /* We need working interrupts for modeset enabling ... */ - drm_irq_install(dev); + drm_irq_install(dev, dev->pdev->irq); intel_modeset_init_hw(dev); drm_modeset_lock_all(dev); - drm_mode_config_reset(dev); intel_modeset_setup_hw_state(dev, true); drm_modeset_unlock_all(dev); @@ -668,7 +657,7 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) intel_hpd_init(dev); dev_priv->enable_hotplug_processing = true; /* Config may have changed between suspend and resume */ - intel_resume_hotplug(dev); + drm_helper_hpd_irq_event(dev); } intel_opregion_init(dev); @@ -685,16 +674,12 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings) schedule_work(&dev_priv->console_resume_work); } - /* Undo what we did at i915_drm_freeze so the refcount goes back to the - * expected level. */ - hsw_enable_package_c8(dev_priv); - mutex_lock(&dev_priv->modeset_restore_lock); dev_priv->modeset_restore = MODESET_DONE; mutex_unlock(&dev_priv->modeset_restore_lock); intel_runtime_pm_put(dev_priv); - return error; + return 0; } static int i915_drm_thaw(struct drm_device *dev) @@ -705,19 +690,33 @@ static int i915_drm_thaw(struct drm_device *dev) return __i915_drm_thaw(dev, true); } -int i915_resume(struct drm_device *dev) +static int i915_resume_early(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; + /* + * We have a resume ordering issue with the snd-hda driver also + * requiring our device to be power up. Due to the lack of a + * parent/child relationship we currently solve this with an early + * resume hook. + * + * FIXME: This should be solved with a special hdmi sink device or + * similar so that power domains can be employed. + */ if (pci_enable_device(dev->pdev)) return -EIO; pci_set_master(dev->pdev); + return i915_drm_thaw_early(dev); +} + +int i915_resume(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + /* * Platforms with opregion should have sane BIOS, older ones (gen3 and * earlier) need to restore the GTT mappings since the BIOS might clear @@ -731,6 +730,14 @@ int i915_resume(struct drm_device *dev) return 0; } +static int i915_resume_legacy(struct drm_device *dev) +{ + i915_resume_early(dev); + i915_resume(dev); + + return 0; +} + /** * i915_reset - reset chip after a hang * @dev: drm device to reset @@ -748,11 +755,11 @@ int i915_resume(struct drm_device *dev) */ int i915_reset(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; bool simulated; int ret; - if (!i915_try_reset) + if (!i915.reset) return 0; mutex_lock(&dev->struct_mutex); @@ -805,8 +812,21 @@ int i915_reset(struct drm_device *dev) return ret; } - drm_irq_uninstall(dev); - drm_irq_install(dev); + /* + * FIXME: This races pretty badly against concurrent holders of + * ring interrupts. This is possible since we've started to drop + * dev->struct_mutex in select places when waiting for the gpu. + */ + + /* + * rps/rc6 re-init is necessary to restore state lost after the + * reset and the re-install of gt irqs. Skip for ironlake per + * previous concerns that it doesn't respond well to some forms + * of re-init after reset. + */ + if (INTEL_INFO(dev)->gen > 5) + intel_reset_gt_powersave(dev); + intel_hpd_init(dev); } else { mutex_unlock(&dev->struct_mutex); @@ -820,7 +840,7 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct intel_device_info *intel_info = (struct intel_device_info *) ent->driver_data; - if (IS_PRELIMINARY_HW(intel_info) && !i915_preliminary_hw_support) { + if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) { DRM_INFO("This hardware requires preliminary hardware support.\n" "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n"); return -ENODEV; @@ -851,7 +871,6 @@ static int i915_pm_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct drm_device *drm_dev = pci_get_drvdata(pdev); - int error; if (!drm_dev || !drm_dev->dev_private) { dev_err(dev, "DRM not initialized, aborting suspend.\n"); @@ -861,9 +880,25 @@ static int i915_pm_suspend(struct device *dev) if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - error = i915_drm_freeze(drm_dev); - if (error) - return error; + return i915_drm_freeze(drm_dev); +} + +static int i915_pm_suspend_late(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + + /* + * We have a suspedn ordering issue with the snd-hda driver also + * requiring our device to be power up. Due to the lack of a + * parent/child relationship we currently solve this with an late + * suspend hook. + * + * FIXME: This should be solved with a special hdmi sink device or + * similar so that power domains can be employed. + */ + if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; pci_disable_device(pdev); pci_set_power_state(pdev, PCI_D3hot); @@ -871,6 +906,14 @@ static int i915_pm_suspend(struct device *dev) return 0; } +static int i915_pm_resume_early(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + + return i915_resume_early(drm_dev); +} + static int i915_pm_resume(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -892,6 +935,14 @@ static int i915_pm_freeze(struct device *dev) return i915_drm_freeze(drm_dev); } +static int i915_pm_thaw_early(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + + return i915_drm_thaw_early(drm_dev); +} + static int i915_pm_thaw(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -908,17 +959,453 @@ static int i915_pm_poweroff(struct device *dev) return i915_drm_freeze(drm_dev); } -static int i915_runtime_suspend(struct device *device) +static int hsw_runtime_suspend(struct drm_i915_private *dev_priv) +{ + hsw_enable_pc8(dev_priv); + + return 0; +} + +static int snb_runtime_resume(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + + intel_init_pch_refclk(dev); + + return 0; +} + +static int hsw_runtime_resume(struct drm_i915_private *dev_priv) +{ + hsw_disable_pc8(dev_priv); + + return 0; +} + +/* + * Save all Gunit registers that may be lost after a D3 and a subsequent + * S0i[R123] transition. The list of registers needing a save/restore is + * defined in the VLV2_S0IXRegs document. This documents marks all Gunit + * registers in the following way: + * - Driver: saved/restored by the driver + * - Punit : saved/restored by the Punit firmware + * - No, w/o marking: no need to save/restore, since the register is R/O or + * used internally by the HW in a way that doesn't depend + * keeping the content across a suspend/resume. + * - Debug : used for debugging + * + * We save/restore all registers marked with 'Driver', with the following + * exceptions: + * - Registers out of use, including also registers marked with 'Debug'. + * These have no effect on the driver's operation, so we don't save/restore + * them to reduce the overhead. + * - Registers that are fully setup by an initialization function called from + * the resume path. For example many clock gating and RPS/RC6 registers. + * - Registers that provide the right functionality with their reset defaults. + * + * TODO: Except for registers that based on the above 3 criteria can be safely + * ignored, we save/restore all others, practically treating the HW context as + * a black-box for the driver. Further investigation is needed to reduce the + * saved/restored registers even further, by following the same 3 criteria. + */ +static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) +{ + struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; + int i; + + /* GAM 0x4000-0x4770 */ + s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); + s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); + s->arb_mode = I915_READ(ARB_MODE); + s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); + s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS_BASE + i * 4); + + s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); + s->gfx_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); + + s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); + s->ecochk = I915_READ(GAM_ECOCHK); + s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); + s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); + + s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); + + /* MBC 0x9024-0x91D0, 0x8500 */ + s->g3dctl = I915_READ(VLV_G3DCTL); + s->gsckgctl = I915_READ(VLV_GSCKGCTL); + s->mbctl = I915_READ(GEN6_MBCTL); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + s->ucgctl1 = I915_READ(GEN6_UCGCTL1); + s->ucgctl3 = I915_READ(GEN6_UCGCTL3); + s->rcgctl1 = I915_READ(GEN6_RCGCTL1); + s->rcgctl2 = I915_READ(GEN6_RCGCTL2); + s->rstctl = I915_READ(GEN6_RSTCTL); + s->misccpctl = I915_READ(GEN7_MISCCPCTL); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + s->gfxpause = I915_READ(GEN6_GFXPAUSE); + s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); + s->rpdeuc = I915_READ(GEN6_RPDEUC); + s->ecobus = I915_READ(ECOBUS); + s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); + s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); + s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); + s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); + s->rcedata = I915_READ(VLV_RCEDATA); + s->spare2gh = I915_READ(VLV_SPAREG2H); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + s->gt_imr = I915_READ(GTIMR); + s->gt_ier = I915_READ(GTIER); + s->pm_imr = I915_READ(GEN6_PMIMR); + s->pm_ier = I915_READ(GEN6_PMIER); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH_BASE + i * 4); + + /* GT SA CZ domain, 0x100000-0x138124 */ + s->tilectl = I915_READ(TILECTL); + s->gt_fifoctl = I915_READ(GTFIFOCTL); + s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); + s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + s->pmwgicz = I915_READ(VLV_PMWGICZ); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + s->gu_ctl0 = I915_READ(VLV_GU_CTL0); + s->gu_ctl1 = I915_READ(VLV_GU_CTL1); + s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); + + /* + * Not saving any of: + * DFT, 0x9800-0x9EC0 + * SARB, 0xB000-0xB1FC + * GAC, 0x5208-0x524C, 0x14000-0x14C000 + * PCI CFG + */ +} + +static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) +{ + struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state; + u32 val; + int i; + + /* GAM 0x4000-0x4770 */ + I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); + I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); + I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); + I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); + I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + I915_WRITE(GEN7_LRA_LIMITS_BASE + i * 4, s->lra_limits[i]); + + I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); + I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->gfx_max_req_count); + + I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); + I915_WRITE(GAM_ECOCHK, s->ecochk); + I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); + I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); + + I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); + + /* MBC 0x9024-0x91D0, 0x8500 */ + I915_WRITE(VLV_G3DCTL, s->g3dctl); + I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); + I915_WRITE(GEN6_MBCTL, s->mbctl); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); + I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); + I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); + I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); + I915_WRITE(GEN6_RSTCTL, s->rstctl); + I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); + I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); + I915_WRITE(GEN6_RPDEUC, s->rpdeuc); + I915_WRITE(ECOBUS, s->ecobus); + I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); + I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); + I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); + I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); + I915_WRITE(VLV_RCEDATA, s->rcedata); + I915_WRITE(VLV_SPAREG2H, s->spare2gh); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + I915_WRITE(GTIMR, s->gt_imr); + I915_WRITE(GTIER, s->gt_ier); + I915_WRITE(GEN6_PMIMR, s->pm_imr); + I915_WRITE(GEN6_PMIER, s->pm_ier); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + I915_WRITE(GEN7_GT_SCRATCH_BASE + i * 4, s->gt_scratch[i]); + + /* GT SA CZ domain, 0x100000-0x138124 */ + I915_WRITE(TILECTL, s->tilectl); + I915_WRITE(GTFIFOCTL, s->gt_fifoctl); + /* + * Preserve the GT allow wake and GFX force clock bit, they are not + * be restored, as they are used to control the s0ix suspend/resume + * sequence by the caller. + */ + val = I915_READ(VLV_GTLC_WAKE_CTRL); + val &= VLV_GTLC_ALLOWWAKEREQ; + val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; + I915_WRITE(VLV_GTLC_WAKE_CTRL, val); + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + val &= VLV_GFX_CLK_FORCE_ON_BIT; + val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); + + I915_WRITE(VLV_PMWGICZ, s->pmwgicz); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); + I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); + I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); +} + +int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) +{ + u32 val; + int err; + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + WARN_ON(!!(val & VLV_GFX_CLK_FORCE_ON_BIT) == force_on); + +#define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT) + /* Wait for a previous force-off to settle */ + if (force_on) { + err = wait_for(!COND, 20); + if (err) { + DRM_ERROR("timeout waiting for GFX clock force-off (%08x)\n", + I915_READ(VLV_GTLC_SURVIVABILITY_REG)); + return err; + } + } + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + val &= ~VLV_GFX_CLK_FORCE_ON_BIT; + if (force_on) + val |= VLV_GFX_CLK_FORCE_ON_BIT; + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); + + if (!force_on) + return 0; + + err = wait_for(COND, 20); + if (err) + DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", + I915_READ(VLV_GTLC_SURVIVABILITY_REG)); + + return err; +#undef COND +} + +static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) +{ + u32 val; + int err = 0; + + val = I915_READ(VLV_GTLC_WAKE_CTRL); + val &= ~VLV_GTLC_ALLOWWAKEREQ; + if (allow) + val |= VLV_GTLC_ALLOWWAKEREQ; + I915_WRITE(VLV_GTLC_WAKE_CTRL, val); + POSTING_READ(VLV_GTLC_WAKE_CTRL); + +#define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \ + allow) + err = wait_for(COND, 1); + if (err) + DRM_ERROR("timeout disabling GT waking\n"); + return err; +#undef COND +} + +static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, + bool wait_for_on) +{ + u32 mask; + u32 val; + int err; + + mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; + val = wait_for_on ? mask : 0; +#define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val) + if (COND) + return 0; + + DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n", + wait_for_on ? "on" : "off", + I915_READ(VLV_GTLC_PW_STATUS)); + + /* + * RC6 transitioning can be delayed up to 2 msec (see + * valleyview_enable_rps), use 3 msec for safety. + */ + err = wait_for(COND, 3); + if (err) + DRM_ERROR("timeout waiting for GT wells to go %s\n", + wait_for_on ? "on" : "off"); + + return err; +#undef COND +} + +static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) +{ + if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) + return; + + DRM_ERROR("GT register access while GT waking disabled\n"); + I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); +} + +static int vlv_runtime_suspend(struct drm_i915_private *dev_priv) +{ + u32 mask; + int err; + + /* + * Bspec defines the following GT well on flags as debug only, so + * don't treat them as hard failures. + */ + (void)vlv_wait_for_gt_wells(dev_priv, false); + + mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; + WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); + + vlv_check_no_gt_access(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, true); + if (err) + goto err1; + + err = vlv_allow_gt_wake(dev_priv, false); + if (err) + goto err2; + vlv_save_gunit_s0ix_state(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, false); + if (err) + goto err2; + + return 0; + +err2: + /* For safety always re-enable waking and disable gfx clock forcing */ + vlv_allow_gt_wake(dev_priv, true); +err1: + vlv_force_gfx_clock(dev_priv, false); + + return err; +} + +static int vlv_runtime_resume(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + int err; + int ret; + + /* + * If any of the steps fail just try to continue, that's the best we + * can do at this point. Return the first error code (which will also + * leave RPM permanently disabled). + */ + ret = vlv_force_gfx_clock(dev_priv, true); + + vlv_restore_gunit_s0ix_state(dev_priv); + + err = vlv_allow_gt_wake(dev_priv, true); + if (!ret) + ret = err; + + err = vlv_force_gfx_clock(dev_priv, false); + if (!ret) + ret = err; + + vlv_check_no_gt_access(dev_priv); + + intel_init_clock_gating(dev); + i915_gem_restore_fences(dev); + + return ret; +} + +static int intel_runtime_suspend(struct device *device) { struct pci_dev *pdev = to_pci_dev(device); struct drm_device *dev = pci_get_drvdata(pdev); struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev)))) + return -ENODEV; WARN_ON(!HAS_RUNTIME_PM(dev)); + assert_force_wake_inactive(dev_priv); DRM_DEBUG_KMS("Suspending device\n"); + /* + * We could deadlock here in case another thread holding struct_mutex + * calls RPM suspend concurrently, since the RPM suspend will wait + * first for this RPM suspend to finish. In this case the concurrent + * RPM resume will be followed by its RPM suspend counterpart. Still + * for consistency return -EAGAIN, which will reschedule this suspend. + */ + if (!mutex_trylock(&dev->struct_mutex)) { + DRM_DEBUG_KMS("device lock contention, deffering suspend\n"); + /* + * Bump the expiration timestamp, otherwise the suspend won't + * be rescheduled. + */ + pm_runtime_mark_last_busy(device); + + return -EAGAIN; + } + /* + * We are safe here against re-faults, since the fault handler takes + * an RPM reference. + */ i915_gem_release_all_mmaps(dev_priv); + mutex_unlock(&dev->struct_mutex); + + /* + * rps.work can't be rearmed here, since we get here only after making + * sure the GPU is idle and the RPS freq is set to the minimum. See + * intel_mark_idle(). + */ + cancel_work_sync(&dev_priv->rps.work); + intel_runtime_pm_disable_interrupts(dev); + + if (IS_GEN6(dev)) { + ret = 0; + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + ret = hsw_runtime_suspend(dev_priv); + } else if (IS_VALLEYVIEW(dev)) { + ret = vlv_runtime_suspend(dev_priv); + } else { + ret = -ENODEV; + WARN_ON(1); + } + + if (ret) { + DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret); + intel_runtime_pm_restore_interrupts(dev); + + return ret; + } del_timer_sync(&dev_priv->gpu_error.hangcheck_timer); dev_priv->pm.suspended = true; @@ -932,14 +1419,16 @@ static int i915_runtime_suspend(struct device *device) */ intel_opregion_notify_adapter(dev, PCI_D1); + DRM_DEBUG_KMS("Device suspended\n"); return 0; } -static int i915_runtime_resume(struct device *device) +static int intel_runtime_resume(struct device *device) { struct pci_dev *pdev = to_pci_dev(device); struct drm_device *dev = pci_get_drvdata(pdev); struct drm_i915_private *dev_priv = dev->dev_private; + int ret; WARN_ON(!HAS_RUNTIME_PM(dev)); @@ -948,18 +1437,48 @@ static int i915_runtime_resume(struct device *device) intel_opregion_notify_adapter(dev, PCI_D0); dev_priv->pm.suspended = false; - return 0; + if (IS_GEN6(dev)) { + ret = snb_runtime_resume(dev_priv); + } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + ret = hsw_runtime_resume(dev_priv); + } else if (IS_VALLEYVIEW(dev)) { + ret = vlv_runtime_resume(dev_priv); + } else { + WARN_ON(1); + ret = -ENODEV; + } + + /* + * No point of rolling back things in case of an error, as the best + * we can do is to hope that things will still work (and disable RPM). + */ + i915_gem_init_swizzling(dev); + gen6_update_ring_freq(dev); + + intel_runtime_pm_restore_interrupts(dev); + intel_reset_gt_powersave(dev); + + if (ret) + DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret); + else + DRM_DEBUG_KMS("Device resumed\n"); + + return ret; } static const struct dev_pm_ops i915_pm_ops = { .suspend = i915_pm_suspend, + .suspend_late = i915_pm_suspend_late, + .resume_early = i915_pm_resume_early, .resume = i915_pm_resume, .freeze = i915_pm_freeze, + .thaw_early = i915_pm_thaw_early, .thaw = i915_pm_thaw, .poweroff = i915_pm_poweroff, + .restore_early = i915_pm_resume_early, .restore = i915_pm_resume, - .runtime_suspend = i915_runtime_suspend, - .runtime_resume = i915_runtime_resume, + .runtime_suspend = intel_runtime_suspend, + .runtime_resume = intel_runtime_resume, }; static const struct vm_operations_struct i915_gem_vm_ops = { @@ -999,7 +1518,7 @@ static struct drm_driver driver = { /* Used in place of i915_pm_ops for non-DRIVER_MODESET */ .suspend = i915_suspend, - .resume = i915_resume, + .resume = i915_resume_legacy, .device_is_agp = i915_driver_device_is_agp, .master_create = i915_master_create, @@ -1051,14 +1570,14 @@ static int __init i915_init(void) * the default behavior. */ #if defined(CONFIG_DRM_I915_KMS) - if (i915_modeset != 0) + if (i915.modeset != 0) driver.driver_features |= DRIVER_MODESET; #endif - if (i915_modeset == 1) + if (i915.modeset == 1) driver.driver_features |= DRIVER_MODESET; #ifdef CONFIG_VGA_CONSOLE - if (vgacon_text_force() && i915_modeset == -1) + if (vgacon_text_force() && i915.modeset == -1) driver.driver_features &= ~DRIVER_MODESET; #endif @@ -1066,6 +1585,7 @@ static int __init i915_init(void) driver.get_vblank_timestamp = NULL; #ifndef CONFIG_DRM_I915_UMS /* Silently fail loading to not upset userspace. */ + DRM_DEBUG_DRIVER("KMS and UMS disabled.\n"); return 0; #endif } diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 4a2bf8e3f73..374f964323a 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -35,11 +35,13 @@ #include "i915_reg.h" #include "intel_bios.h" #include "intel_ringbuffer.h" +#include "i915_gem_gtt.h" #include <linux/io-mapping.h> #include <linux/i2c.h> #include <linux/i2c-algo-bit.h> #include <drm/intel-gtt.h> #include <linux/backlight.h> +#include <linux/hashtable.h> #include <linux/intel-iommu.h> #include <linux/kref.h> #include <linux/pm_qos.h> @@ -58,7 +60,8 @@ enum pipe { PIPE_A = 0, PIPE_B, PIPE_C, - I915_MAX_PIPES + _PIPE_EDP, + I915_MAX_PIPES = _PIPE_EDP }; #define pipe_name(p) ((p) + 'A') @@ -66,7 +69,8 @@ enum transcoder { TRANSCODER_A = 0, TRANSCODER_B, TRANSCODER_C, - TRANSCODER_EDP = 0xF, + TRANSCODER_EDP, + I915_MAX_TRANSCODERS }; #define transcoder_name(t) ((t) + 'A') @@ -77,7 +81,7 @@ enum plane { }; #define plane_name(p) ((p) + 'A') -#define sprite_name(p, s) ((p) * dev_priv->num_plane + (s) + 'A') +#define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A') enum port { PORT_A = 0, @@ -89,7 +93,7 @@ enum port { }; #define port_name(p) ((p) + 'A') -#define I915_NUM_PHYS_VLV 1 +#define I915_NUM_PHYS_VLV 2 enum dpio_channel { DPIO_CH0, @@ -112,6 +116,17 @@ enum intel_display_power_domain { POWER_DOMAIN_TRANSCODER_B, POWER_DOMAIN_TRANSCODER_C, POWER_DOMAIN_TRANSCODER_EDP, + POWER_DOMAIN_PORT_DDI_A_2_LANES, + POWER_DOMAIN_PORT_DDI_A_4_LANES, + POWER_DOMAIN_PORT_DDI_B_2_LANES, + POWER_DOMAIN_PORT_DDI_B_4_LANES, + POWER_DOMAIN_PORT_DDI_C_2_LANES, + POWER_DOMAIN_PORT_DDI_C_4_LANES, + POWER_DOMAIN_PORT_DDI_D_2_LANES, + POWER_DOMAIN_PORT_DDI_D_4_LANES, + POWER_DOMAIN_PORT_DSI, + POWER_DOMAIN_PORT_CRT, + POWER_DOMAIN_PORT_OTHER, POWER_DOMAIN_VGA, POWER_DOMAIN_AUDIO, POWER_DOMAIN_INIT, @@ -119,8 +134,6 @@ enum intel_display_power_domain { POWER_DOMAIN_NUM, }; -#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) - #define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A) #define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \ ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER) @@ -128,14 +141,6 @@ enum intel_display_power_domain { ((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \ (tran) + POWER_DOMAIN_TRANSCODER_A) -#define HSW_ALWAYS_ON_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PIPE_A) | \ - BIT(POWER_DOMAIN_TRANSCODER_EDP)) -#define BDW_ALWAYS_ON_POWER_DOMAINS ( \ - BIT(POWER_DOMAIN_PIPE_A) | \ - BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ - BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER)) - enum hpd_pin { HPD_NONE = 0, HPD_PORT_A = HPD_NONE, /* PORT_A is internal */ @@ -157,12 +162,24 @@ enum hpd_pin { I915_GEM_DOMAIN_VERTEX) #define for_each_pipe(p) for ((p) = 0; (p) < INTEL_INFO(dev)->num_pipes; (p)++) +#define for_each_sprite(p, s) for ((s) = 0; (s) < INTEL_INFO(dev)->num_sprites[(p)]; (s)++) + +#define for_each_crtc(dev, crtc) \ + list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + +#define for_each_intel_crtc(dev, intel_crtc) \ + list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \ if ((intel_encoder)->base.crtc == (__crtc)) +#define for_each_connector_on_encoder(dev, __encoder, intel_connector) \ + list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \ + if ((intel_connector)->base.encoder == (__encoder)) + struct drm_i915_private; +struct i915_mmu_object; enum intel_dpll_id { DPLL_ID_PRIVATE = -1, /* non-shared dpll in use */ @@ -234,18 +251,6 @@ struct intel_ddi_plls { #define WATCH_LISTS 0 #define WATCH_GTT 0 -#define I915_GEM_PHYS_CURSOR_0 1 -#define I915_GEM_PHYS_CURSOR_1 2 -#define I915_GEM_PHYS_OVERLAY_REGS 3 -#define I915_MAX_PHYS_OBJECT (I915_GEM_PHYS_OVERLAY_REGS) - -struct drm_i915_gem_phys_object { - int id; - struct page **page_list; - drm_dma_handle_t *handle; - struct drm_i915_gem_object *cur_obj; -}; - struct opregion_header; struct opregion_acpi; struct opregion_swsci; @@ -295,53 +300,86 @@ struct intel_display_error_state; struct drm_i915_error_state { struct kref ref; + struct timeval time; + + char error_msg[128]; + u32 reset_count; + u32 suspend_count; + + /* Generic register state */ u32 eir; u32 pgtbl_er; u32 ier; u32 ccid; u32 derrmr; u32 forcewake; - bool waiting[I915_NUM_RINGS]; - u32 pipestat[I915_MAX_PIPES]; - u32 tail[I915_NUM_RINGS]; - u32 head[I915_NUM_RINGS]; - u32 ctl[I915_NUM_RINGS]; - u32 ipeir[I915_NUM_RINGS]; - u32 ipehr[I915_NUM_RINGS]; - u32 instdone[I915_NUM_RINGS]; - u32 acthd[I915_NUM_RINGS]; - u32 semaphore_mboxes[I915_NUM_RINGS][I915_NUM_RINGS - 1]; - u32 semaphore_seqno[I915_NUM_RINGS][I915_NUM_RINGS - 1]; - u32 rc_psmi[I915_NUM_RINGS]; /* sleep state */ - /* our own tracking of ring head and tail */ - u32 cpu_ring_head[I915_NUM_RINGS]; - u32 cpu_ring_tail[I915_NUM_RINGS]; u32 error; /* gen6+ */ u32 err_int; /* gen7 */ - u32 bbstate[I915_NUM_RINGS]; - u32 instpm[I915_NUM_RINGS]; - u32 instps[I915_NUM_RINGS]; - u32 extra_instdone[I915_NUM_INSTDONE_REG]; - u32 seqno[I915_NUM_RINGS]; - u64 bbaddr[I915_NUM_RINGS]; - u32 fault_reg[I915_NUM_RINGS]; u32 done_reg; - u32 faddr[I915_NUM_RINGS]; + u32 gac_eco; + u32 gam_ecochk; + u32 gab_ctl; + u32 gfx_mode; + u32 extra_instdone[I915_NUM_INSTDONE_REG]; u64 fence[I915_MAX_NUM_FENCES]; - struct timeval time; + struct intel_overlay_error_state *overlay; + struct intel_display_error_state *display; + struct drm_i915_error_ring { bool valid; + /* Software tracked state */ + bool waiting; + int hangcheck_score; + enum intel_ring_hangcheck_action hangcheck_action; + int num_requests; + + /* our own tracking of ring head and tail */ + u32 cpu_ring_head; + u32 cpu_ring_tail; + + u32 semaphore_seqno[I915_NUM_RINGS - 1]; + + /* Register state */ + u32 tail; + u32 head; + u32 ctl; + u32 hws; + u32 ipeir; + u32 ipehr; + u32 instdone; + u32 bbstate; + u32 instpm; + u32 instps; + u32 seqno; + u64 bbaddr; + u64 acthd; + u32 fault_reg; + u64 faddr; + u32 rc_psmi; /* sleep state */ + u32 semaphore_mboxes[I915_NUM_RINGS - 1]; + struct drm_i915_error_object { int page_count; u32 gtt_offset; u32 *pages[0]; - } *ringbuffer, *batchbuffer, *ctx; + } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; + struct drm_i915_error_request { long jiffies; u32 seqno; u32 tail; } *requests; - int num_requests; + + struct { + u32 gfx_mode; + union { + u64 pdp[4]; + u32 pp_dir_base; + }; + } vm_info; + + pid_t pid; + char comm[TASK_COMM_LEN]; } ring[I915_NUM_RINGS]; struct drm_i915_error_buffer { u32 size; @@ -355,18 +393,17 @@ struct drm_i915_error_state { u32 tiling:2; u32 dirty:1; u32 purgeable:1; + u32 userptr:1; s32 ring:4; u32 cache_level:3; } **active_bo, **pinned_bo; + u32 *active_bo_count, *pinned_bo_count; - struct intel_overlay_error_state *overlay; - struct intel_display_error_state *display; - int hangcheck_score[I915_NUM_RINGS]; - enum intel_ring_hangcheck_action hangcheck_action[I915_NUM_RINGS]; }; struct intel_connector; struct intel_crtc_config; +struct intel_plane_config; struct intel_crtc; struct intel_limit; struct dpll; @@ -405,6 +442,8 @@ struct drm_i915_display_funcs { * fills out the pipe-config with the hw state. */ bool (*get_pipe_config)(struct intel_crtc *, struct intel_crtc_config *); + void (*get_plane_config)(struct intel_crtc *, + struct intel_plane_config *); int (*crtc_mode_set)(struct drm_crtc *crtc, int x, int y, struct drm_framebuffer *old_fb); @@ -419,9 +458,11 @@ struct drm_i915_display_funcs { int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags); - int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb, - int x, int y); + void (*update_primary_plane)(struct drm_crtc *crtc, + struct drm_framebuffer *fb, + int x, int y); void (*hpd_irq_setup)(struct drm_device *dev); /* clock updates for mode set */ /* cursor updates */ @@ -469,7 +510,7 @@ struct intel_uncore { unsigned fw_rendercount; unsigned fw_mediacount; - struct delayed_work force_wake_work; + struct timer_list force_wake_timer; }; #define DEV_INFO_FOR_EACH_FLAG(func, sep) \ @@ -504,9 +545,17 @@ struct intel_uncore { struct intel_device_info { u32 display_mmio_offset; u8 num_pipes:3; + u8 num_sprites[I915_MAX_PIPES]; u8 gen; u8 ring_mask; /* Rings supported by the HW */ DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON); + /* Register offsets for the various display pipes and transcoders */ + int pipe_offsets[I915_MAX_TRANSCODERS]; + int trans_offsets[I915_MAX_TRANSCODERS]; + int dpll_offsets[I915_MAX_PIPES]; + int dpll_md_offsets[I915_MAX_PIPES]; + int palette_offsets[I915_MAX_PIPES]; + int cursor_offsets[I915_MAX_PIPES]; }; #undef DEFINE_FLAG @@ -522,138 +571,6 @@ enum i915_cache_level { I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */ }; -typedef uint32_t gen6_gtt_pte_t; - -struct i915_address_space { - struct drm_mm mm; - struct drm_device *dev; - struct list_head global_link; - unsigned long start; /* Start offset always 0 for dri2 */ - size_t total; /* size addr space maps (ex. 2GB for ggtt) */ - - struct { - dma_addr_t addr; - struct page *page; - } scratch; - - /** - * List of objects currently involved in rendering. - * - * Includes buffers having the contents of their GPU caches - * flushed, not necessarily primitives. last_rendering_seqno - * represents when the rendering involved will be completed. - * - * A reference is held on the buffer while on this list. - */ - struct list_head active_list; - - /** - * LRU list of objects which are not in the ringbuffer and - * are ready to unbind, but are still in the GTT. - * - * last_rendering_seqno is 0 while an object is in this list. - * - * A reference is not held on the buffer while on this list, - * as merely being GTT-bound shouldn't prevent its being - * freed, and we'll pull it off the list in the free path. - */ - struct list_head inactive_list; - - /* FIXME: Need a more generic return type */ - gen6_gtt_pte_t (*pte_encode)(dma_addr_t addr, - enum i915_cache_level level, - bool valid); /* Create a valid PTE */ - void (*clear_range)(struct i915_address_space *vm, - unsigned int first_entry, - unsigned int num_entries, - bool use_scratch); - void (*insert_entries)(struct i915_address_space *vm, - struct sg_table *st, - unsigned int first_entry, - enum i915_cache_level cache_level); - void (*cleanup)(struct i915_address_space *vm); -}; - -/* The Graphics Translation Table is the way in which GEN hardware translates a - * Graphics Virtual Address into a Physical Address. In addition to the normal - * collateral associated with any va->pa translations GEN hardware also has a - * portion of the GTT which can be mapped by the CPU and remain both coherent - * and correct (in cases like swizzling). That region is referred to as GMADR in - * the spec. - */ -struct i915_gtt { - struct i915_address_space base; - size_t stolen_size; /* Total size of stolen memory */ - - unsigned long mappable_end; /* End offset that we can CPU map */ - struct io_mapping *mappable; /* Mapping to our CPU mappable region */ - phys_addr_t mappable_base; /* PA of our GMADR */ - - /** "Graphics Stolen Memory" holds the global PTEs */ - void __iomem *gsm; - - bool do_idle_maps; - - int mtrr; - - /* global gtt ops */ - int (*gtt_probe)(struct drm_device *dev, size_t *gtt_total, - size_t *stolen, phys_addr_t *mappable_base, - unsigned long *mappable_end); -}; -#define gtt_total_entries(gtt) ((gtt).base.total >> PAGE_SHIFT) - -struct i915_hw_ppgtt { - struct i915_address_space base; - unsigned num_pd_entries; - union { - struct page **pt_pages; - struct page *gen8_pt_pages; - }; - struct page *pd_pages; - int num_pd_pages; - int num_pt_pages; - union { - uint32_t pd_offset; - dma_addr_t pd_dma_addr[4]; - }; - union { - dma_addr_t *pt_dma_addr; - dma_addr_t *gen8_pt_dma_addr[4]; - }; - int (*enable)(struct drm_device *dev); -}; - -/** - * A VMA represents a GEM BO that is bound into an address space. Therefore, a - * VMA's presence cannot be guaranteed before binding, or after unbinding the - * object into/from the address space. - * - * To make things as simple as possible (ie. no refcounting), a VMA's lifetime - * will always be <= an objects lifetime. So object refcounting should cover us. - */ -struct i915_vma { - struct drm_mm_node node; - struct drm_i915_gem_object *obj; - struct i915_address_space *vm; - - /** This object's place on the active/inactive lists */ - struct list_head mm_list; - - struct list_head vma_link; /* Link in the object's VMA list */ - - /** This vma's place in the batchbuffer or on the eviction list */ - struct list_head exec_list; - - /** - * Used for performing relocations during execbuffer insertion. - */ - struct hlist_node exec_node; - unsigned long exec_handle; - struct drm_i915_gem_exec_object2 *exec_entry; - -}; - struct i915_ctx_hang_stats { /* This context had batch pending when hang was declared */ unsigned batch_pending; @@ -670,15 +587,16 @@ struct i915_ctx_hang_stats { /* This must match up with the value previously used for execbuf2.rsvd1. */ #define DEFAULT_CONTEXT_ID 0 -struct i915_hw_context { +struct intel_context { struct kref ref; int id; bool is_initialized; uint8_t remap_slice; struct drm_i915_file_private *file_priv; - struct intel_ring_buffer *ring; + struct intel_engine_cs *last_ring; struct drm_i915_gem_object *obj; struct i915_ctx_hang_stats hang_stats; + struct i915_address_space *vm; struct list_head link; }; @@ -713,6 +631,10 @@ struct i915_fbc { } no_fbc_reason; }; +struct i915_drrs { + struct intel_connector *connector; +}; + struct i915_psr { bool sink_support; bool source_ok; @@ -734,6 +656,7 @@ enum intel_sbi_destination { #define QUIRK_PIPEA_FORCE (1<<0) #define QUIRK_LVDS_SSC_DISABLE (1<<1) #define QUIRK_INVERT_BRIGHTNESS (1<<2) +#define QUIRK_BACKLIGHT_PRESENT (1<<3) struct intel_fbdev; struct intel_fbc_work; @@ -831,11 +754,7 @@ struct i915_suspend_saved_registers { u32 savePFIT_CONTROL; u32 save_palette_a[256]; u32 save_palette_b[256]; - u32 saveDPFC_CB_BASE; - u32 saveFBC_CFB_BASE; - u32 saveFBC_LL_BASE; u32 saveFBC_CONTROL; - u32 saveFBC_CONTROL2; u32 saveIER; u32 saveIIR; u32 saveIMR; @@ -900,20 +819,90 @@ struct i915_suspend_saved_registers { u32 savePCH_PORT_HOTPLUG; }; +struct vlv_s0ix_state { + /* GAM */ + u32 wr_watermark; + u32 gfx_prio_ctrl; + u32 arb_mode; + u32 gfx_pend_tlb0; + u32 gfx_pend_tlb1; + u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; + u32 media_max_req_count; + u32 gfx_max_req_count; + u32 render_hwsp; + u32 ecochk; + u32 bsd_hwsp; + u32 blt_hwsp; + u32 tlb_rd_addr; + + /* MBC */ + u32 g3dctl; + u32 gsckgctl; + u32 mbctl; + + /* GCP */ + u32 ucgctl1; + u32 ucgctl3; + u32 rcgctl1; + u32 rcgctl2; + u32 rstctl; + u32 misccpctl; + + /* GPM */ + u32 gfxpause; + u32 rpdeuhwtc; + u32 rpdeuc; + u32 ecobus; + u32 pwrdwnupctl; + u32 rp_down_timeout; + u32 rp_deucsw; + u32 rcubmabdtmr; + u32 rcedata; + u32 spare2gh; + + /* Display 1 CZ domain */ + u32 gt_imr; + u32 gt_ier; + u32 pm_imr; + u32 pm_ier; + u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; + + /* GT SA CZ domain */ + u32 tilectl; + u32 gt_fifoctl; + u32 gtlc_wake_ctrl; + u32 gtlc_survive; + u32 pmwgicz; + + /* Display 2 CZ domain */ + u32 gu_ctl0; + u32 gu_ctl1; + u32 clock_gate_dis2; +}; + struct intel_gen6_power_mgmt { /* work and pm_iir are protected by dev_priv->irq_lock */ struct work_struct work; u32 pm_iir; - /* The below variables an all the rps hw state are protected by - * dev->struct mutext. */ - u8 cur_delay; - u8 min_delay; - u8 max_delay; - u8 rpe_delay; - u8 rp1_delay; - u8 rp0_delay; - u8 hw_max; + /* Frequencies are stored in potentially platform dependent multiples. + * In other words, *_freq needs to be multiplied by X to be interesting. + * Soft limits are those which are used for the dynamic reclocking done + * by the driver (raise frequencies under heavy loads, and lower for + * lighter loads). Hard limits are those imposed by the hardware. + * + * A distinction is made for overclocking, which is never enabled by + * default, and is considered to be above the hard limit if it's + * possible at all. + */ + u8 cur_freq; /* Current frequency (cached, may not == HW) */ + u8 min_freq_softlimit; /* Minimum frequency permitted by the driver */ + u8 max_freq_softlimit; /* Max frequency permitted by the driver */ + u8 max_freq; /* Maximum frequency, RP0 if not overclocking */ + u8 min_freq; /* AKA RPn. Minimum frequency */ + u8 efficient_freq; /* AKA RPe. Pre-determined balanced frequency */ + u8 rp1_freq; /* "less than" RP0 power/freqency */ + u8 rp0_freq; /* Non-overclocked max frequency. */ int last_adj; enum { LOW_POWER, BETWEEN, HIGH_POWER } power; @@ -953,18 +942,47 @@ struct intel_ilk_power_mgmt { struct drm_i915_gem_object *renderctx; }; +struct drm_i915_private; +struct i915_power_well; + +struct i915_power_well_ops { + /* + * Synchronize the well's hw state to match the current sw state, for + * example enable/disable it based on the current refcount. Called + * during driver init and resume time, possibly after first calling + * the enable/disable handlers. + */ + void (*sync_hw)(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well); + /* + * Enable the well and resources that depend on it (for example + * interrupts located on the well). Called after the 0->1 refcount + * transition. + */ + void (*enable)(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well); + /* + * Disable the well and resources that depend on it. Called after + * the 1->0 refcount transition. + */ + void (*disable)(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well); + /* Returns the hw enabled state. */ + bool (*is_enabled)(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well); +}; + /* Power well structure for haswell */ struct i915_power_well { const char *name; bool always_on; /* power well enable/disable usage count */ int count; + /* cached hw enabled state */ + bool hw_enabled; unsigned long domains; - void *data; - void (*set)(struct drm_device *dev, struct i915_power_well *power_well, - bool enable); - bool (*is_enabled)(struct drm_device *dev, - struct i915_power_well *power_well); + unsigned long data; + const struct i915_power_well_ops *ops; }; struct i915_power_domains { @@ -973,6 +991,7 @@ struct i915_power_domains { * time are on. They are kept on until after the first modeset. */ bool init_power_on; + bool initializing; int power_well_count; struct mutex lock; @@ -1031,7 +1050,8 @@ struct i915_gem_mm { /** PPGTT used for aliasing the PPGTT with the GTT */ struct i915_hw_ppgtt *aliasing_ppgtt; - struct shrinker inactive_shrinker; + struct notifier_block oom_notifier; + struct shrinker shrinker; bool shrinker_no_lock_stealing; /** LRU list of objects with fence regs on them. */ @@ -1061,14 +1081,22 @@ struct i915_gem_mm { */ bool interruptible; + /** + * Is the GPU currently considered idle, or busy executing userspace + * requests? Whilst idle, we attempt to power down the hardware and + * display clocks. In order to reduce the effect on performance, there + * is a slight delay before we do so. + */ + bool busy; + + /* the indicator for dispatch video commands on two BSD rings */ + int bsd_ring_dispatch_index; + /** Bit 6 swizzling required for X tiling */ uint32_t bit_6_swizzle_x; /** Bit 6 swizzling required for Y tiling */ uint32_t bit_6_swizzle_y; - /* storage for physical objects */ - struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT]; - /* accounting, useful for userland debugging */ spinlock_t object_stat_lock; size_t object_memory; @@ -1139,8 +1167,12 @@ struct i915_gpu_error { */ wait_queue_head_t reset_queue; - /* For gpu hang simulation. */ - unsigned int stop_rings; + /* Userspace knobs for gpu hang simulation; + * combines both a ring mask, and extra flags + */ + u32 stop_rings; +#define I915_STOP_RING_ALLOW_BAN (1 << 31) +#define I915_STOP_RING_ALLOW_WARN (1 << 30) /* For missed irq/seqno simulation. */ unsigned int test_irq_rings; @@ -1160,6 +1192,12 @@ struct ddi_vbt_port_info { uint8_t supports_dp:1; }; +enum drrs_support_type { + DRRS_NOT_SUPPORTED = 0, + STATIC_DRRS_SUPPORT = 1, + SEAMLESS_DRRS_SUPPORT = 2 +}; + struct intel_vbt_data { struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ @@ -1172,9 +1210,12 @@ struct intel_vbt_data { unsigned int lvds_use_ssc:1; unsigned int display_clock_mode:1; unsigned int fdi_rx_polarity_inverted:1; + unsigned int has_mipi:1; int lvds_ssc_freq; unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */ + enum drrs_support_type drrs_type; + /* eDP */ int edp_rate; int edp_lanes; @@ -1187,12 +1228,20 @@ struct intel_vbt_data { struct { u16 pwm_freq_hz; + bool present; bool active_low_pwm; } backlight; /* MIPI DSI */ struct { + u16 port; u16 panel_id; + struct mipi_config *config; + struct mipi_pps_data *pps; + u8 seq_version; + u32 size; + u8 *data; + u8 *sequence[MIPI_SEQ_MAX]; } dsi; int crt_ddc_pin; @@ -1226,76 +1275,31 @@ struct ilk_wm_values { }; /* - * This struct tracks the state needed for the Package C8+ feature. - * - * Package states C8 and deeper are really deep PC states that can only be - * reached when all the devices on the system allow it, so even if the graphics - * device allows PC8+, it doesn't mean the system will actually get to these - * states. - * - * Our driver only allows PC8+ when all the outputs are disabled, the power well - * is disabled and the GPU is idle. When these conditions are met, we manually - * do the other conditions: disable the interrupts, clocks and switch LCPLL - * refclk to Fclk. + * This struct helps tracking the state needed for runtime PM, which puts the + * device in PCI D3 state. Notice that when this happens, nothing on the + * graphics device works, even register access, so we don't get interrupts nor + * anything else. * - * When we really reach PC8 or deeper states (not just when we allow it) we lose - * the state of some registers, so when we come back from PC8+ we need to - * restore this state. We don't get into PC8+ if we're not in RC6, so we don't - * need to take care of the registers kept by RC6. + * Every piece of our code that needs to actually touch the hardware needs to + * either call intel_runtime_pm_get or call intel_display_power_get with the + * appropriate power domain. * - * The interrupt disabling is part of the requirements. We can only leave the - * PCH HPD interrupts enabled. If we're in PC8+ and we get another interrupt we - * can lock the machine. - * - * Ideally every piece of our code that needs PC8+ disabled would call - * hsw_disable_package_c8, which would increment disable_count and prevent the - * system from reaching PC8+. But we don't have a symmetric way to do this for - * everything, so we have the requirements_met and gpu_idle variables. When we - * switch requirements_met or gpu_idle to true we decrease disable_count, and - * increase it in the opposite case. The requirements_met variable is true when - * all the CRTCs, encoders and the power well are disabled. The gpu_idle - * variable is true when the GPU is idle. - * - * In addition to everything, we only actually enable PC8+ if disable_count - * stays at zero for at least some seconds. This is implemented with the - * enable_work variable. We do this so we don't enable/disable PC8 dozens of - * consecutive times when all screens are disabled and some background app - * queries the state of our connectors, or we have some application constantly - * waking up to use the GPU. Only after the enable_work function actually - * enables PC8+ the "enable" variable will become true, which means that it can - * be false even if disable_count is 0. + * Our driver uses the autosuspend delay feature, which means we'll only really + * suspend if we stay with zero refcount for a certain amount of time. The + * default value is currently very conservative (see intel_init_runtime_pm), but + * it can be changed with the standard runtime PM files from sysfs. * * The irqs_disabled variable becomes true exactly after we disable the IRQs and * goes back to false exactly before we reenable the IRQs. We use this variable * to check if someone is trying to enable/disable IRQs while they're supposed * to be disabled. This shouldn't happen and we'll print some error messages in - * case it happens, but if it actually happens we'll also update the variables - * inside struct regsave so when we restore the IRQs they will contain the - * latest expected values. + * case it happens. * - * For more, read "Display Sequences for Package C8" on our documentation. + * For more, read the Documentation/power/runtime_pm.txt. */ -struct i915_package_c8 { - bool requirements_met; - bool gpu_idle; - bool irqs_disabled; - /* Only true after the delayed work task actually enables it. */ - bool enabled; - int disable_count; - struct mutex lock; - struct delayed_work enable_work; - - struct { - uint32_t deimr; - uint32_t sdeimr; - uint32_t gtimr; - uint32_t gtier; - uint32_t gen6_pmimr; - } regsave; -}; - struct i915_runtime_pm { bool suspended; + bool irqs_disabled; }; enum intel_pipe_crc_source { @@ -1328,11 +1332,11 @@ struct intel_pipe_crc { wait_queue_head_t wq; }; -typedef struct drm_i915_private { +struct drm_i915_private { struct drm_device *dev; struct kmem_cache *slab; - const struct intel_device_info *info; + const struct intel_device_info info; int relative_constants_mode; @@ -1352,20 +1356,23 @@ typedef struct drm_i915_private { */ uint32_t gpio_mmio_base; + /* MMIO base address for MIPI regs */ + uint32_t mipi_mmio_base; + wait_queue_head_t gmbus_wait_queue; struct pci_dev *bridge_dev; - struct intel_ring_buffer ring[I915_NUM_RINGS]; + struct intel_engine_cs ring[I915_NUM_RINGS]; uint32_t last_seqno, next_seqno; drm_dma_handle_t *status_page_dmah; struct resource mch_res; - atomic_t irq_received; - /* protects the irq masks */ spinlock_t irq_lock; + bool display_irqs_enabled; + /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */ struct pm_qos_request pm_qos; @@ -1379,6 +1386,8 @@ typedef struct drm_i915_private { }; u32 gt_irq_mask; u32 pm_irq_mask; + u32 pm_rps_events; + u32 pipestat_irq_mask[I915_MAX_PIPES]; struct work_struct hotplug_work; bool enable_hotplug_processing; @@ -1394,9 +1403,8 @@ typedef struct drm_i915_private { u32 hpd_event_bits; struct timer_list hotplug_reenable_timer; - int num_plane; - struct i915_fbc fbc; + struct i915_drrs drrs; struct intel_opregion opregion; struct intel_vbt_data vbt; @@ -1414,6 +1422,7 @@ typedef struct drm_i915_private { int num_fence_regs; /* 8 on pre-965, 16 otherwise */ unsigned int fsb_freq, mem_freq, is_ddr3; + unsigned int vlv_cdclk_freq; /** * wq - Driver workqueue for GEM. @@ -1437,16 +1446,19 @@ typedef struct drm_i915_private { struct mutex modeset_restore_lock; struct list_head vm_list; /* Global list of all address spaces */ - struct i915_gtt gtt; /* VMA representing the global address space */ + struct i915_gtt gtt; /* VM representing the global address space */ struct i915_gem_mm mm; +#if defined(CONFIG_MMU_NOTIFIER) + DECLARE_HASHTABLE(mmu_notifiers, 7); +#endif /* Kernel Modesetting */ struct sdvo_device_mapping sdvo_mappings[2]; - struct drm_crtc *plane_to_crtc_mapping[3]; - struct drm_crtc *pipe_to_crtc_mapping[3]; + struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; + struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; wait_queue_head_t pending_flip_queue; #ifdef CONFIG_DEBUG_FS @@ -1506,7 +1518,9 @@ typedef struct drm_i915_private { u32 fdi_rx_config; + u32 suspend_count; struct i915_suspend_saved_registers regfile; + struct vlv_s0ix_state vlv_s0ix_state; struct { /* @@ -1525,8 +1539,6 @@ typedef struct drm_i915_private { struct ilk_wm_values hw; } wm; - struct i915_package_c8 pc8; - struct i915_runtime_pm pm; /* Old dri1 support infrastructure, beware the dragons ya fools entering @@ -1534,7 +1546,12 @@ typedef struct drm_i915_private { struct i915_dri1_state dri1; /* Old ums support infrastructure, same warning applies. */ struct i915_ums_state ums; -} drm_i915_private_t; + + /* + * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch + * will be rejected. Instead look for a better place. + */ +}; static inline struct drm_i915_private *to_i915(const struct drm_device *dev) { @@ -1571,6 +1588,8 @@ struct drm_i915_gem_object_ops { */ int (*get_pages)(struct drm_i915_gem_object *); void (*put_pages)(struct drm_i915_gem_object *); + int (*dmabuf_export)(struct drm_i915_gem_object *); + void (*release)(struct drm_i915_gem_object *); }; struct drm_i915_gem_object { @@ -1627,18 +1646,6 @@ struct drm_i915_gem_object { */ unsigned int fence_dirty:1; - /** How many users have pinned this object in GTT space. The following - * users can each hold at most one reference: pwrite/pread, pin_ioctl - * (via user_pin_count), execbuffer (objects are not allowed multiple - * times for the same batchbuffer), and the framebuffer code. When - * switching/pageflipping, the framebuffer code has at most two buffers - * pinned per crtc. - * - * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 - * bits with absolutely no headroom. So use 4 bits. */ - unsigned int pin_count:4; -#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf - /** * Is the object at the current location in the gtt mappable and * fenceable? Used to avoid costly recalculations. @@ -1673,7 +1680,7 @@ struct drm_i915_gem_object { void *dma_buf_vmapping; int vmapping_count; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; /** Breadcrumb of last rendering to the buffer. */ uint32_t last_read_seqno; @@ -1695,10 +1702,21 @@ struct drm_i915_gem_object { struct drm_file *pin_filp; /** for phy allocated objects */ - struct drm_i915_gem_phys_object *phys_obj; -}; -#define to_gem_object(obj) (&((struct drm_i915_gem_object *)(obj))->base) + drm_dma_handle_t *phys_handle; + union { + struct i915_gem_userptr { + uintptr_t ptr; + unsigned read_only :1; + unsigned workers :4; +#define I915_GEM_USERPTR_MAX_WORKERS 15 + + struct mm_struct *mm; + struct i915_mmu_object *mn; + struct work_struct *work; + } userptr; + }; +}; #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base) /** @@ -1713,7 +1731,7 @@ struct drm_i915_gem_object { */ struct drm_i915_gem_request { /** On Which ring this request was generated */ - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; /** GEM sequence number associated with this request. */ uint32_t seqno; @@ -1725,7 +1743,7 @@ struct drm_i915_gem_request { u32 tail; /** Context related to this request */ - struct i915_hw_context *ctx; + struct intel_context *ctx; /** Batch buffer related to this request if any */ struct drm_i915_gem_object *batch_obj; @@ -1743,6 +1761,7 @@ struct drm_i915_gem_request { struct drm_i915_file_private { struct drm_i915_private *dev_priv; + struct drm_file *file; struct { spinlock_t lock; @@ -1751,11 +1770,101 @@ struct drm_i915_file_private { } mm; struct idr context_idr; - struct i915_ctx_hang_stats hang_stats; atomic_t rps_wait_boost; + struct intel_engine_cs *bsd_ring; +}; + +/* + * A command that requires special handling by the command parser. + */ +struct drm_i915_cmd_descriptor { + /* + * Flags describing how the command parser processes the command. + * + * CMD_DESC_FIXED: The command has a fixed length if this is set, + * a length mask if not set + * CMD_DESC_SKIP: The command is allowed but does not follow the + * standard length encoding for the opcode range in + * which it falls + * CMD_DESC_REJECT: The command is never allowed + * CMD_DESC_REGISTER: The command should be checked against the + * register whitelist for the appropriate ring + * CMD_DESC_MASTER: The command is allowed if the submitting process + * is the DRM master + */ + u32 flags; +#define CMD_DESC_FIXED (1<<0) +#define CMD_DESC_SKIP (1<<1) +#define CMD_DESC_REJECT (1<<2) +#define CMD_DESC_REGISTER (1<<3) +#define CMD_DESC_BITMASK (1<<4) +#define CMD_DESC_MASTER (1<<5) + + /* + * The command's unique identification bits and the bitmask to get them. + * This isn't strictly the opcode field as defined in the spec and may + * also include type, subtype, and/or subop fields. + */ + struct { + u32 value; + u32 mask; + } cmd; + + /* + * The command's length. The command is either fixed length (i.e. does + * not include a length field) or has a length field mask. The flag + * CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has + * a length mask. All command entries in a command table must include + * length information. + */ + union { + u32 fixed; + u32 mask; + } length; + + /* + * Describes where to find a register address in the command to check + * against the ring's register whitelist. Only valid if flags has the + * CMD_DESC_REGISTER bit set. + */ + struct { + u32 offset; + u32 mask; + } reg; + +#define MAX_CMD_DESC_BITMASKS 3 + /* + * Describes command checks where a particular dword is masked and + * compared against an expected value. If the command does not match + * the expected value, the parser rejects it. Only valid if flags has + * the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero + * are valid. + * + * If the check specifies a non-zero condition_mask then the parser + * only performs the check when the bits specified by condition_mask + * are non-zero. + */ + struct { + u32 offset; + u32 mask; + u32 expected; + u32 condition_offset; + u32 condition_mask; + } bits[MAX_CMD_DESC_BITMASKS]; +}; + +/* + * A table of commands requiring special handling by the command parser. + * + * Each ring has an array of tables. Each table consists of an array of command + * descriptors, which must be sorted with command opcodes in ascending order. + */ +struct drm_i915_cmd_table { + const struct drm_i915_cmd_descriptor *table; + int count; }; -#define INTEL_INFO(dev) (to_i915(dev)->info) +#define INTEL_INFO(dev) (&to_i915(dev)->info) #define IS_I830(dev) ((dev)->pdev->device == 0x3577) #define IS_845G(dev) ((dev)->pdev->device == 0x2562) @@ -1782,8 +1891,9 @@ struct drm_i915_file_private { (dev)->pdev->device == 0x0106 || \ (dev)->pdev->device == 0x010A) #define IS_VALLEYVIEW(dev) (INTEL_INFO(dev)->is_valleyview) +#define IS_CHERRYVIEW(dev) (INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev)) #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell) -#define IS_BROADWELL(dev) (INTEL_INFO(dev)->gen == 8) +#define IS_BROADWELL(dev) (!INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev)) #define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) #define IS_HSW_EARLY_SDV(dev) (IS_HASWELL(dev) && \ ((dev)->pdev->device & 0xFF00) == 0x0C00) @@ -1796,6 +1906,9 @@ struct drm_i915_file_private { #define IS_ULT(dev) (IS_HSW_ULT(dev) || IS_BDW_ULT(dev)) #define IS_HSW_GT3(dev) (IS_HASWELL(dev) && \ ((dev)->pdev->device & 0x00F0) == 0x0020) +/* ULX machines are also considered ULT. */ +#define IS_HSW_ULX(dev) ((dev)->pdev->device == 0x0A0E || \ + (dev)->pdev->device == 0x0A1E) #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary) /* @@ -1816,21 +1929,37 @@ struct drm_i915_file_private { #define BSD_RING (1<<VCS) #define BLT_RING (1<<BCS) #define VEBOX_RING (1<<VECS) -#define HAS_BSD(dev) (INTEL_INFO(dev)->ring_mask & BSD_RING) -#define HAS_BLT(dev) (INTEL_INFO(dev)->ring_mask & BLT_RING) -#define HAS_VEBOX(dev) (INTEL_INFO(dev)->ring_mask & VEBOX_RING) -#define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) -#define HAS_WT(dev) (IS_HASWELL(dev) && to_i915(dev)->ellc_size) +#define BSD2_RING (1<<VCS2) +#define HAS_BSD(dev) (INTEL_INFO(dev)->ring_mask & BSD_RING) +#define HAS_BSD2(dev) (INTEL_INFO(dev)->ring_mask & BSD2_RING) +#define HAS_BLT(dev) (INTEL_INFO(dev)->ring_mask & BLT_RING) +#define HAS_VEBOX(dev) (INTEL_INFO(dev)->ring_mask & VEBOX_RING) +#define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) +#define HAS_WT(dev) ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \ + to_i915(dev)->ellc_size) #define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) #define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->gen >= 6) -#define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >=6 && !IS_VALLEYVIEW(dev)) +#define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >= 6 && \ + (!IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))) +#define HAS_PPGTT(dev) (INTEL_INFO(dev)->gen >= 7 \ + && !IS_GEN8(dev)) +#define USES_PPGTT(dev) intel_enable_ppgtt(dev, false) +#define USES_FULL_PPGTT(dev) intel_enable_ppgtt(dev, true) #define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) #define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) /* Early gen2 have a totally busted CS tlb and require pinned batches. */ #define HAS_BROKEN_CS_TLB(dev) (IS_I830(dev) || IS_845G(dev)) +/* + * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts + * even when in MSI mode. This results in spurious interrupt warnings if the + * legacy irq no. is shared with another device. The kernel then disables that + * interrupt source and so prevents the other device from working properly. + */ +#define HAS_AUX_IRQ(dev) (INTEL_INFO(dev)->gen >= 5) +#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->gen >= 5) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. @@ -1852,8 +1981,8 @@ struct drm_i915_file_private { #define HAS_DDI(dev) (INTEL_INFO(dev)->has_ddi) #define HAS_FPGA_DBG_UNCLAIMED(dev) (INTEL_INFO(dev)->has_fpga_dbg) #define HAS_PSR(dev) (IS_HASWELL(dev) || IS_BROADWELL(dev)) -#define HAS_PC8(dev) (IS_HASWELL(dev)) /* XXX HSW:ULX */ -#define HAS_RUNTIME_PM(dev) (IS_HASWELL(dev)) +#define HAS_RUNTIME_PM(dev) (IS_GEN6(dev) || IS_HASWELL(dev) || \ + IS_BROADWELL(dev) || IS_VALLEYVIEW(dev)) #define INTEL_PCH_DEVICE_ID_MASK 0xff00 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 @@ -1879,32 +2008,41 @@ struct drm_i915_file_private { extern const struct drm_ioctl_desc i915_ioctls[]; extern int i915_max_ioctl; -extern unsigned int i915_fbpercrtc __always_unused; -extern int i915_panel_ignore_lid __read_mostly; -extern unsigned int i915_powersave __read_mostly; -extern int i915_semaphores __read_mostly; -extern unsigned int i915_lvds_downclock __read_mostly; -extern int i915_lvds_channel_mode __read_mostly; -extern int i915_panel_use_ssc __read_mostly; -extern int i915_vbt_sdvo_panel_type __read_mostly; -extern int i915_enable_rc6 __read_mostly; -extern int i915_enable_fbc __read_mostly; -extern bool i915_enable_hangcheck __read_mostly; -extern int i915_enable_ppgtt __read_mostly; -extern int i915_enable_psr __read_mostly; -extern unsigned int i915_preliminary_hw_support __read_mostly; -extern int i915_disable_power_well __read_mostly; -extern int i915_enable_ips __read_mostly; -extern bool i915_fastboot __read_mostly; -extern int i915_enable_pc8 __read_mostly; -extern int i915_pc8_timeout __read_mostly; -extern bool i915_prefault_disable __read_mostly; extern int i915_suspend(struct drm_device *dev, pm_message_t state); extern int i915_resume(struct drm_device *dev); extern int i915_master_create(struct drm_device *dev, struct drm_master *master); extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master); +/* i915_params.c */ +struct i915_params { + int modeset; + int panel_ignore_lid; + unsigned int powersave; + int semaphores; + unsigned int lvds_downclock; + int lvds_channel_mode; + int panel_use_ssc; + int vbt_sdvo_panel_type; + int enable_rc6; + int enable_fbc; + int enable_ppgtt; + int enable_psr; + unsigned int preliminary_hw_support; + int disable_power_well; + int enable_ips; + int invert_brightness; + int enable_cmd_parser; + /* leave bools at the end to not create holes */ + bool enable_hangcheck; + bool fastboot; + bool prefault_disable; + bool reset; + bool disable_display; + bool disable_vtd_wa; +}; +extern struct i915_params i915 __read_mostly; + /* i915_dma.c */ void i915_update_dri1_breadcrumb(struct drm_device *dev); extern void i915_kernel_lost_context(struct drm_device * dev); @@ -1930,13 +2068,18 @@ extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); +int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); extern void intel_console_resume(struct work_struct *work); /* i915_irq.c */ void i915_queue_hangcheck(struct drm_device *dev); -void i915_handle_error(struct drm_device *dev, bool wedged); +__printf(3, 4) +void i915_handle_error(struct drm_device *dev, bool wedged, + const char *fmt, ...); +void gen6_set_pm_mask(struct drm_i915_private *dev_priv, u32 pm_iir, + int new_delay); extern void intel_irq_init(struct drm_device *dev); extern void intel_hpd_init(struct drm_device *dev); @@ -1947,10 +2090,15 @@ extern void intel_uncore_check_errors(struct drm_device *dev); extern void intel_uncore_fini(struct drm_device *dev); void -i915_enable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask); +i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, + u32 status_mask); void -i915_disable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask); +i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, + u32 status_mask); + +void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv); +void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv); /* i915_gem.c */ int i915_gem_init_ioctl(struct drm_device *dev, void *data, @@ -1995,6 +2143,9 @@ int i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); +int i915_gem_init_userptr(struct drm_device *dev); +int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_wait_ioctl(struct drm_device *dev, void *data, @@ -2006,22 +2157,29 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops); struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, size_t size); +void i915_init_vm(struct drm_i915_private *dev_priv, + struct i915_address_space *vm); void i915_gem_free_object(struct drm_gem_object *obj); void i915_gem_vma_destroy(struct i915_vma *vma); +#define PIN_MAPPABLE 0x1 +#define PIN_NONBLOCK 0x2 +#define PIN_GLOBAL 0x4 +#define PIN_OFFSET_BIAS 0x8 +#define PIN_OFFSET_MASK (~4095) int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj, struct i915_address_space *vm, uint32_t alignment, - bool map_and_fenceable, - bool nonblocking); -void i915_gem_object_unpin(struct drm_i915_gem_object *obj); + uint64_t flags); int __must_check i915_vma_unbind(struct i915_vma *vma); -int __must_check i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj); int i915_gem_object_put_pages(struct drm_i915_gem_object *obj); void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); void i915_gem_lastclose(struct drm_device *dev); +int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, + int *needs_clflush); + int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj); static inline struct page *i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n) { @@ -2045,9 +2203,9 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); int i915_gem_object_sync(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *to); + struct intel_engine_cs *to); void i915_vma_move_to_active(struct i915_vma *vma, - struct intel_ring_buffer *ring); + struct intel_engine_cs *ring); int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); @@ -2067,29 +2225,14 @@ int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj); int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj); -static inline bool -i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) -{ - if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - dev_priv->fence_regs[obj->fence_reg].pin_count++; - return true; - } else - return false; -} +bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj); +void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj); -static inline void -i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) -{ - if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); - dev_priv->fence_regs[obj->fence_reg].pin_count--; - } -} +struct drm_i915_gem_request * +i915_gem_find_active_request(struct intel_engine_cs *ring); bool i915_gem_retire_requests(struct drm_device *dev); -void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring); +void i915_gem_retire_requests_ring(struct intel_engine_cs *ring); int __must_check i915_gem_check_wedge(struct i915_gpu_error *error, bool interruptible); static inline bool i915_reset_in_progress(struct i915_gpu_error *error) @@ -2108,23 +2251,35 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error) return ((atomic_read(&error->reset_counter) & ~I915_WEDGED) + 1) / 2; } +static inline bool i915_stop_ring_allow_ban(struct drm_i915_private *dev_priv) +{ + return dev_priv->gpu_error.stop_rings == 0 || + dev_priv->gpu_error.stop_rings & I915_STOP_RING_ALLOW_BAN; +} + +static inline bool i915_stop_ring_allow_warn(struct drm_i915_private *dev_priv) +{ + return dev_priv->gpu_error.stop_rings == 0 || + dev_priv->gpu_error.stop_rings & I915_STOP_RING_ALLOW_WARN; +} + void i915_gem_reset(struct drm_device *dev); bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj); int __must_check i915_gem_init(struct drm_device *dev); int __must_check i915_gem_init_hw(struct drm_device *dev); -int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice); +int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice); void i915_gem_init_swizzling(struct drm_device *dev); void i915_gem_cleanup_ringbuffer(struct drm_device *dev); int __must_check i915_gpu_idle(struct drm_device *dev); int __must_check i915_gem_suspend(struct drm_device *dev); -int __i915_add_request(struct intel_ring_buffer *ring, +int __i915_add_request(struct intel_engine_cs *ring, struct drm_file *file, struct drm_i915_gem_object *batch_obj, u32 *seqno); #define i915_add_request(ring, seqno) \ __i915_add_request(ring, NULL, NULL, seqno) -int __must_check i915_wait_seqno(struct intel_ring_buffer *ring, +int __must_check i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno); int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); int __must_check @@ -2135,15 +2290,10 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write); int __must_check i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, - struct intel_ring_buffer *pipelined); + struct intel_engine_cs *pipelined); void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj); -int i915_gem_attach_phys_object(struct drm_device *dev, - struct drm_i915_gem_object *obj, - int id, +int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align); -void i915_gem_detach_phys_object(struct drm_device *dev, - struct drm_i915_gem_object *obj); -void i915_gem_free_all_phys_object(struct drm_device *dev); int i915_gem_open(struct drm_device *dev, struct drm_file *file); void i915_gem_release(struct drm_device *dev, struct drm_file *file); @@ -2178,6 +2328,13 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, struct i915_address_space *vm); struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj); +static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) { + struct i915_vma *vma; + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (vma->pin_count > 0) + return true; + return false; +} /* Some GGTT VM helpers */ #define obj_to_ggtt(obj) \ @@ -2209,77 +2366,73 @@ i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj) static inline int __must_check i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj, uint32_t alignment, - bool map_and_fenceable, - bool nonblocking) + unsigned flags) +{ + return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL); +} + +static inline int +i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj) { - return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, - map_and_fenceable, nonblocking); + return i915_vma_unbind(i915_gem_obj_to_ggtt(obj)); } +void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj); + /* i915_gem_context.c */ +#define ctx_to_ppgtt(ctx) container_of((ctx)->vm, struct i915_hw_ppgtt, base) int __must_check i915_gem_context_init(struct drm_device *dev); void i915_gem_context_fini(struct drm_device *dev); +void i915_gem_context_reset(struct drm_device *dev); +int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); +int i915_gem_context_enable(struct drm_i915_private *dev_priv); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); -int i915_switch_context(struct intel_ring_buffer *ring, - struct drm_file *file, int to_id); +int i915_switch_context(struct intel_engine_cs *ring, + struct intel_context *to); +struct intel_context * +i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id); void i915_gem_context_free(struct kref *ctx_ref); -static inline void i915_gem_context_reference(struct i915_hw_context *ctx) +static inline void i915_gem_context_reference(struct intel_context *ctx) { kref_get(&ctx->ref); } -static inline void i915_gem_context_unreference(struct i915_hw_context *ctx) +static inline void i915_gem_context_unreference(struct intel_context *ctx) { kref_put(&ctx->ref, i915_gem_context_free); } -struct i915_ctx_hang_stats * __must_check -i915_gem_context_get_hang_stats(struct drm_device *dev, - struct drm_file *file, - u32 id); +static inline bool i915_gem_context_is_default(const struct intel_context *c) +{ + return c->id == DEFAULT_CONTEXT_ID; +} + int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); -/* i915_gem_gtt.c */ -void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev); -void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_object *obj, - enum i915_cache_level cache_level); -void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_object *obj); - -void i915_check_and_clear_faults(struct drm_device *dev); -void i915_gem_suspend_gtt_mappings(struct drm_device *dev); -void i915_gem_restore_gtt_mappings(struct drm_device *dev); -int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj); -void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, - enum i915_cache_level cache_level); -void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj); -void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); -void i915_gem_init_global_gtt(struct drm_device *dev); -void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start, - unsigned long mappable_end, unsigned long end); -int i915_gem_gtt_init(struct drm_device *dev); -static inline void i915_gem_chipset_flush(struct drm_device *dev) -{ - if (INTEL_INFO(dev)->gen < 6) - intel_gtt_chipset_flush(); -} - - +/* i915_gem_render_state.c */ +int i915_gem_render_state_init(struct intel_engine_cs *ring); /* i915_gem_evict.c */ int __must_check i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm, int min_size, unsigned alignment, unsigned cache_level, - bool mappable, - bool nonblock); + unsigned long start, + unsigned long end, + unsigned flags); int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle); int i915_gem_evict_everything(struct drm_device *dev); +/* belongs in i915_gem_gtt.h */ +static inline void i915_gem_chipset_flush(struct drm_device *dev) +{ + if (INTEL_INFO(dev)->gen < 6) + intel_gtt_chipset_flush(); +} + /* i915_gem_stolen.c */ int i915_gem_init_stolen(struct drm_device *dev); int i915_gem_stolen_setup_compression(struct drm_device *dev, int size); @@ -2297,7 +2450,7 @@ void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj); /* i915_gem_tiling.c */ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) { - drm_i915_private_t *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && obj->tiling_mode != I915_TILING_NONE; @@ -2335,7 +2488,8 @@ static inline void i915_error_state_buf_release( { kfree(eb->buf); } -void i915_capture_error_state(struct drm_device *dev); +void i915_capture_error_state(struct drm_device *dev, bool wedge, + const char *error_msg); void i915_error_state_get(struct drm_device *dev, struct i915_error_state_file_priv *error_priv); void i915_error_state_put(struct i915_error_state_file_priv *error_priv); @@ -2344,6 +2498,16 @@ void i915_destroy_error_state(struct drm_device *dev); void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone); const char *i915_cache_level_str(int type); +/* i915_cmd_parser.c */ +int i915_cmd_parser_get_version(void); +int i915_cmd_parser_init_ring(struct intel_engine_cs *ring); +void i915_cmd_parser_fini_ring(struct intel_engine_cs *ring); +bool i915_needs_cmd_parser(struct intel_engine_cs *ring); +int i915_parse_cmds(struct intel_engine_cs *ring, + struct drm_i915_gem_object *batch_obj, + u32 batch_start_offset, + bool is_master); + /* i915_suspend.c */ extern int i915_save_state(struct drm_device *dev); extern int i915_restore_state(struct drm_device *dev); @@ -2417,10 +2581,12 @@ extern void intel_modeset_suspend_hw(struct drm_device *dev); extern void intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); +extern void intel_connector_unregister(struct intel_connector *); extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); extern void intel_modeset_setup_hw_state(struct drm_device *dev, bool force_restore); extern void i915_redisable_vga(struct drm_device *dev); +extern void i915_redisable_vga_power_on(struct drm_device *dev); extern bool intel_fbc_enabled(struct drm_device *dev); extern void intel_disable_fbc(struct drm_device *dev); extern bool ironlake_set_drps(struct drm_device *dev, u8 val); @@ -2455,6 +2621,7 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, */ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine); void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine); +void assert_force_wake_inactive(struct drm_i915_private *dev_priv); int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val); int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val); @@ -2485,20 +2652,6 @@ void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val); int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val); -void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine); -void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine); - -#define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \ - (((reg) >= 0x2000 && (reg) < 0x4000) ||\ - ((reg) >= 0x5000 && (reg) < 0x8000) ||\ - ((reg) >= 0xB000 && (reg) < 0x12000) ||\ - ((reg) >= 0x2E000 && (reg) < 0x30000)) - -#define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)\ - (((reg) >= 0x12000 && (reg) < 0x14000) ||\ - ((reg) >= 0x22000 && (reg) < 0x24000) ||\ - ((reg) >= 0x30000 && (reg) < 0x40000)) - #define FORCEWAKE_RENDER (1 << 0) #define FORCEWAKE_MEDIA (1 << 1) #define FORCEWAKE_ALL (FORCEWAKE_RENDER | FORCEWAKE_MEDIA) @@ -2517,9 +2670,26 @@ void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine); #define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false) #define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false) +/* Be very careful with read/write 64-bit values. On 32-bit machines, they + * will be implemented using 2 32-bit writes in an arbitrary order with + * an arbitrary delay between them. This can cause the hardware to + * act upon the intermediate value, possibly leading to corruption and + * machine death. You have been warned. + */ #define I915_WRITE64(reg, val) dev_priv->uncore.funcs.mmio_writeq(dev_priv, (reg), (val), true) #define I915_READ64(reg) dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true) +#define I915_READ64_2x32(lower_reg, upper_reg) ({ \ + u32 upper = I915_READ(upper_reg); \ + u32 lower = I915_READ(lower_reg); \ + u32 tmp = I915_READ(upper_reg); \ + if (upper != tmp) { \ + upper = tmp; \ + lower = I915_READ(lower_reg); \ + WARN_ON(I915_READ(upper_reg) != upper); \ + } \ + (u64)upper << 32 | lower; }) + #define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) #define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) @@ -2558,4 +2728,31 @@ timespec_to_jiffies_timeout(const struct timespec *value) return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1); } +/* + * If you need to wait X milliseconds between events A and B, but event B + * doesn't happen exactly after event A, you record the timestamp (jiffies) of + * when event A happened, then just before event B you call this function and + * pass the timestamp as the first argument, and X as the second argument. + */ +static inline void +wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) +{ + unsigned long target_jiffies, tmp_jiffies, remaining_jiffies; + + /* + * Don't re-read the value of "jiffies" every time since it may change + * behind our back and break the math. + */ + tmp_jiffies = jiffies; + target_jiffies = timestamp_jiffies + + msecs_to_jiffies_timeout(to_wait_ms); + + if (time_after(target_jiffies, tmp_jiffies)) { + remaining_jiffies = target_jiffies - tmp_jiffies; + while (remaining_jiffies) + remaining_jiffies = + schedule_timeout_uninterruptible(remaining_jiffies); + } +} + #endif diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 00c83615472..d893e4da5dc 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -31,6 +31,7 @@ #include "i915_drv.h" #include "i915_trace.h" #include "intel_drv.h" +#include <linux/oom.h> #include <linux/shmem_fs.h> #include <linux/slab.h> #include <linux/swap.h> @@ -43,16 +44,8 @@ static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *o static __must_check int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, bool readonly); -static __must_check int -i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, - struct i915_address_space *vm, - unsigned alignment, - bool map_and_fenceable, - bool nonblocking); -static int i915_gem_phys_pwrite(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file); +static void +i915_gem_object_retire(struct drm_i915_gem_object *obj); static void i915_gem_write_fence(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj); @@ -60,13 +53,15 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, struct drm_i915_fence_reg *fence, bool enable); -static unsigned long i915_gem_inactive_count(struct shrinker *shrinker, +static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc); -static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker, +static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc); +static int i915_gem_shrinker_oom(struct notifier_block *nb, + unsigned long event, + void *ptr); static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target); static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); -static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); static bool cpu_cache_is_coherent(struct drm_device *dev, enum i915_cache_level level) @@ -204,7 +199,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, pinned = 0; mutex_lock(&dev->struct_mutex); list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) - if (obj->pin_count) + if (i915_gem_obj_is_pinned(obj)) pinned += i915_gem_obj_ggtt_size(obj); mutex_unlock(&dev->struct_mutex); @@ -214,6 +209,128 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, return 0; } +static void i915_gem_object_detach_phys(struct drm_i915_gem_object *obj) +{ + drm_dma_handle_t *phys = obj->phys_handle; + + if (!phys) + return; + + if (obj->madv == I915_MADV_WILLNEED) { + struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; + char *vaddr = phys->vaddr; + int i; + + for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { + struct page *page = shmem_read_mapping_page(mapping, i); + if (!IS_ERR(page)) { + char *dst = kmap_atomic(page); + memcpy(dst, vaddr, PAGE_SIZE); + drm_clflush_virt_range(dst, PAGE_SIZE); + kunmap_atomic(dst); + + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + } + vaddr += PAGE_SIZE; + } + i915_gem_chipset_flush(obj->base.dev); + } + +#ifdef CONFIG_X86 + set_memory_wb((unsigned long)phys->vaddr, phys->size / PAGE_SIZE); +#endif + drm_pci_free(obj->base.dev, phys); + obj->phys_handle = NULL; +} + +int +i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, + int align) +{ + drm_dma_handle_t *phys; + struct address_space *mapping; + char *vaddr; + int i; + + if (obj->phys_handle) { + if ((unsigned long)obj->phys_handle->vaddr & (align -1)) + return -EBUSY; + + return 0; + } + + if (obj->madv != I915_MADV_WILLNEED) + return -EFAULT; + + if (obj->base.filp == NULL) + return -EINVAL; + + /* create a new object */ + phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); + if (!phys) + return -ENOMEM; + + vaddr = phys->vaddr; +#ifdef CONFIG_X86 + set_memory_wc((unsigned long)vaddr, phys->size / PAGE_SIZE); +#endif + mapping = file_inode(obj->base.filp)->i_mapping; + for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { + struct page *page; + char *src; + + page = shmem_read_mapping_page(mapping, i); + if (IS_ERR(page)) { +#ifdef CONFIG_X86 + set_memory_wb((unsigned long)phys->vaddr, phys->size / PAGE_SIZE); +#endif + drm_pci_free(obj->base.dev, phys); + return PTR_ERR(page); + } + + src = kmap_atomic(page); + memcpy(vaddr, src, PAGE_SIZE); + kunmap_atomic(src); + + mark_page_accessed(page); + page_cache_release(page); + + vaddr += PAGE_SIZE; + } + + obj->phys_handle = phys; + return 0; +} + +static int +i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, + struct drm_i915_gem_pwrite *args, + struct drm_file *file_priv) +{ + struct drm_device *dev = obj->base.dev; + void *vaddr = obj->phys_handle->vaddr + args->offset; + char __user *user_data = to_user_ptr(args->data_ptr); + + if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { + unsigned long unwritten; + + /* The physical object once assigned is fixed for the lifetime + * of the obj, so we can safely drop the lock and continue + * to access vaddr. + */ + mutex_unlock(&dev->struct_mutex); + unwritten = copy_from_user(vaddr, user_data, args->size); + mutex_lock(&dev->struct_mutex); + if (unwritten) + return -EFAULT; + } + + i915_gem_chipset_flush(dev); + return 0; +} + void *i915_gem_object_alloc(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -332,6 +449,44 @@ __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, return 0; } +/* + * Pins the specified object's pages and synchronizes the object with + * GPU accesses. Sets needs_clflush to non-zero if the caller should + * flush the object from the CPU cache. + */ +int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, + int *needs_clflush) +{ + int ret; + + *needs_clflush = 0; + + if (!obj->base.filp) + return -EINVAL; + + if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { + /* If we're not in the cpu read domain, set ourself into the gtt + * read domain and manually flush cachelines (if required). This + * optimizes for the case when the gpu will dirty the data + * anyway again before the next pread happens. */ + *needs_clflush = !cpu_cache_is_coherent(obj->base.dev, + obj->cache_level); + ret = i915_gem_object_wait_rendering(obj, true); + if (ret) + return ret; + + i915_gem_object_retire(obj); + } + + ret = i915_gem_object_get_pages(obj); + if (ret) + return ret; + + i915_gem_object_pin_pages(obj); + + return ret; +} + /* Per-page copy function for the shmem pread fastpath. * Flushes invalid cachelines before reading the target if * needs_clflush is set. */ @@ -429,23 +584,10 @@ i915_gem_shmem_pread(struct drm_device *dev, obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { - /* If we're not in the cpu read domain, set ourself into the gtt - * read domain and manually flush cachelines (if required). This - * optimizes for the case when the gpu will dirty the data - * anyway again before the next pread happens. */ - needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); - ret = i915_gem_object_wait_rendering(obj, true); - if (ret) - return ret; - } - - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); if (ret) return ret; - i915_gem_object_pin_pages(obj); - offset = args->offset; for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, @@ -476,7 +618,7 @@ i915_gem_shmem_pread(struct drm_device *dev, mutex_unlock(&dev->struct_mutex); - if (likely(!i915_prefault_disable) && !prefaulted) { + if (likely(!i915.prefault_disable) && !prefaulted) { ret = fault_in_multipages_writeable(user_data, remain); /* Userspace is tricking us, but we've already clobbered * its pages with the prefault and promised to write the @@ -492,12 +634,10 @@ i915_gem_shmem_pread(struct drm_device *dev, mutex_lock(&dev->struct_mutex); -next_page: - mark_page_accessed(page); - if (ret) goto out; +next_page: remain -= page_length; user_data += page_length; offset += page_length; @@ -599,13 +739,13 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_i915_gem_pwrite *args, struct drm_file *file) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; ssize_t remain; loff_t offset, page_base; char __user *user_data; int page_offset, page_length, ret; - ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); + ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK); if (ret) goto out; @@ -651,7 +791,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, } out_unpin: - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); out: return ret; } @@ -677,9 +817,8 @@ shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, if (needs_clflush_before) drm_clflush_virt_range(vaddr + shmem_page_offset, page_length); - ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, - user_data, - page_length); + ret = __copy_from_user_inatomic(vaddr + shmem_page_offset, + user_data, page_length); if (needs_clflush_after) drm_clflush_virt_range(vaddr + shmem_page_offset, page_length); @@ -752,6 +891,8 @@ i915_gem_shmem_pwrite(struct drm_device *dev, ret = i915_gem_object_wait_rendering(obj, false); if (ret) return ret; + + i915_gem_object_retire(obj); } /* Same trick applies to invalidate partially written cachelines read * before writing. */ @@ -813,13 +954,10 @@ i915_gem_shmem_pwrite(struct drm_device *dev, mutex_lock(&dev->struct_mutex); -next_page: - set_page_dirty(page); - mark_page_accessed(page); - if (ret) goto out; +next_page: remain -= page_length; user_data += page_length; offset += page_length; @@ -868,7 +1006,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, args->size)) return -EFAULT; - if (likely(!i915_prefault_disable)) { + if (likely(!i915.prefault_disable)) { ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), args->size); if (ret) @@ -909,8 +1047,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, * pread/pwrite currently are reading and writing from the CPU * perspective, requiring manual detiling by the client. */ - if (obj->phys_obj) { - ret = i915_gem_phys_pwrite(dev, obj, args, file); + if (obj->phys_handle) { + ret = i915_gem_phys_pwrite(obj, args, file); goto out; } @@ -958,7 +1096,7 @@ i915_gem_check_wedge(struct i915_gpu_error *error, * equal. */ static int -i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) +i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno) { int ret; @@ -977,7 +1115,7 @@ static void fake_irq(unsigned long data) } static bool missed_irq(struct drm_i915_private *dev_priv, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings); } @@ -1008,13 +1146,14 @@ static bool can_wait_boost(struct drm_i915_file_private *file_priv) * Returns 0 if the seqno was found within the alloted time. Else returns the * errno with remaining time filled in timeout argument. */ -static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, +static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, unsigned reset_counter, bool interruptible, struct timespec *timeout, struct drm_i915_file_private *file_priv) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; const bool irq_test_in_progress = ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring); struct timespec before, now; @@ -1022,14 +1161,14 @@ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, unsigned long timeout_expire; int ret; - WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n"); + WARN(dev_priv->pm.irqs_disabled, "IRQs disabled\n"); if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) return 0; timeout_expire = timeout ? jiffies + timespec_to_jiffies_timeout(timeout) : 0; - if (dev_priv->info->gen >= 6 && can_wait_boost(file_priv)) { + if (INTEL_INFO(dev)->gen >= 6 && can_wait_boost(file_priv)) { gen6_rps_boost(dev_priv); if (file_priv) mod_delayed_work(dev_priv->wq, @@ -1114,7 +1253,7 @@ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, * request and object lists appropriately for that event. */ int -i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) +i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1139,9 +1278,10 @@ i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) static int i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { - i915_gem_retire_requests_ring(ring); + if (!obj->active) + return 0; /* Manually manage the write flush as we may have not yet * retired the buffer. @@ -1151,7 +1291,6 @@ i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj, * we know we have passed the last write. */ obj->last_write_seqno = 0; - obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; return 0; } @@ -1164,7 +1303,7 @@ static __must_check int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, bool readonly) { - struct intel_ring_buffer *ring = obj->ring; + struct intel_engine_cs *ring = obj->ring; u32 seqno; int ret; @@ -1184,12 +1323,12 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, */ static __must_check int i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, - struct drm_file *file, + struct drm_i915_file_private *file_priv, bool readonly) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = obj->ring; + struct intel_engine_cs *ring = obj->ring; unsigned reset_counter; u32 seqno; int ret; @@ -1211,7 +1350,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); mutex_unlock(&dev->struct_mutex); - ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file->driver_priv); + ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv); mutex_lock(&dev->struct_mutex); if (ret) return ret; @@ -1260,7 +1399,9 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, * We will repeat the flush holding the lock in the normal manner * to catch cases where we are gazumped. */ - ret = i915_gem_object_wait_rendering__nonblocking(obj, file, !write_domain); + ret = i915_gem_object_wait_rendering__nonblocking(obj, + file->driver_priv, + !write_domain); if (ret) goto unref; @@ -1374,7 +1515,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data); struct drm_device *dev = obj->base.dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; pgoff_t page_offset; unsigned long pfn; int ret = 0; @@ -1392,14 +1533,23 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) trace_i915_gem_object_fault(obj, page_offset, true, write); + /* Try to flush the object off the GPU first without holding the lock. + * Upon reacquiring the lock, we will perform our sanity checks and then + * repeat the flush holding the lock in the normal manner to catch cases + * where we are gazumped. + */ + ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write); + if (ret) + goto unlock; + /* Access to snoopable pages through the GTT is incoherent. */ if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { - ret = -EINVAL; + ret = -EFAULT; goto unlock; } /* Now bind it into the GTT if needed */ - ret = i915_gem_obj_ggtt_pin(obj, 0, true, false); + ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE); if (ret) goto unlock; @@ -1420,7 +1570,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) /* Finally, remap it using the new GTT offset */ ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn); unpin: - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); unlock: mutex_unlock(&dev->struct_mutex); out: @@ -1453,6 +1603,7 @@ out: ret = VM_FAULT_OOM; break; case -ENOSPC: + case -EFAULT: ret = VM_FAULT_SIGBUS; break; default: @@ -1465,22 +1616,6 @@ out: return ret; } -void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) -{ - struct i915_vma *vma; - - /* - * Only the global gtt is relevant for gtt memory mappings, so restrict - * list traversal to objects bound into the global address space. Note - * that the active list should be empty, but better safe than sorry. - */ - WARN_ON(!list_empty(&dev_priv->gtt.base.active_list)); - list_for_each_entry(vma, &dev_priv->gtt.base.active_list, mm_list) - i915_gem_release_mmap(vma->obj); - list_for_each_entry(vma, &dev_priv->gtt.base.inactive_list, mm_list) - i915_gem_release_mmap(vma->obj); -} - /** * i915_gem_release_mmap - remove physical page mappings * @obj: obj in question @@ -1501,10 +1636,20 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) if (!obj->fault_mappable) return; - drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping); + drm_vma_node_unmap(&obj->base.vma_node, + obj->base.dev->anon_inode->i_mapping); obj->fault_mappable = false; } +void +i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) +{ + struct drm_i915_gem_object *obj; + + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) + i915_gem_release_mmap(obj); +} + uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) { @@ -1617,8 +1762,8 @@ i915_gem_mmap_gtt(struct drm_file *file, } if (obj->madv != I915_MADV_WILLNEED) { - DRM_ERROR("Attempting to mmap a purgeable buffer\n"); - ret = -EINVAL; + DRM_DEBUG("Attempting to mmap a purgeable buffer\n"); + ret = -EFAULT; goto out; } @@ -1659,12 +1804,16 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); } +static inline int +i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) +{ + return obj->madv == I915_MADV_DONTNEED; +} + /* Immediately discard the backing storage */ static void i915_gem_object_truncate(struct drm_i915_gem_object *obj) { - struct inode *inode; - i915_gem_object_free_mmap_offset(obj); if (obj->base.filp == NULL) @@ -1675,16 +1824,28 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj) * To do this we must instruct the shmfs to drop all of its * backing pages, *now*. */ - inode = file_inode(obj->base.filp); - shmem_truncate_range(inode, 0, (loff_t)-1); - + shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); obj->madv = __I915_MADV_PURGED; } -static inline int -i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) +/* Try to discard unwanted pages */ +static void +i915_gem_object_invalidate(struct drm_i915_gem_object *obj) { - return obj->madv == I915_MADV_DONTNEED; + struct address_space *mapping; + + switch (obj->madv) { + case I915_MADV_DONTNEED: + i915_gem_object_truncate(obj); + case __I915_MADV_PURGED: + return; + } + + if (obj->base.filp == NULL) + return; + + mapping = file_inode(obj->base.filp)->i_mapping, + invalidate_mapping_pages(mapping, 0, (loff_t)-1); } static void @@ -1749,8 +1910,7 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) ops->put_pages(obj); obj->pages = NULL; - if (i915_gem_object_is_purgeable(obj)) - i915_gem_object_truncate(obj); + i915_gem_object_invalidate(obj); return 0; } @@ -1759,58 +1919,58 @@ static unsigned long __i915_gem_shrink(struct drm_i915_private *dev_priv, long target, bool purgeable_only) { - struct list_head still_bound_list; - struct drm_i915_gem_object *obj, *next; + struct list_head still_in_list; + struct drm_i915_gem_object *obj; unsigned long count = 0; - list_for_each_entry_safe(obj, next, - &dev_priv->mm.unbound_list, - global_list) { - if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && - i915_gem_object_put_pages(obj) == 0) { - count += obj->base.size >> PAGE_SHIFT; - if (count >= target) - return count; - } - } - /* - * As we may completely rewrite the bound list whilst unbinding + * As we may completely rewrite the (un)bound list whilst unbinding * (due to retiring requests) we have to strictly process only * one element of the list at the time, and recheck the list * on every iteration. + * + * In particular, we must hold a reference whilst removing the + * object as we may end up waiting for and/or retiring the objects. + * This might release the final reference (held by the active list) + * and result in the object being freed from under us. This is + * similar to the precautions the eviction code must take whilst + * removing objects. + * + * Also note that although these lists do not hold a reference to + * the object we can safely grab one here: The final object + * unreferencing and the bound_list are both protected by the + * dev->struct_mutex and so we won't ever be able to observe an + * object on the bound_list with a reference count equals 0. */ - INIT_LIST_HEAD(&still_bound_list); + INIT_LIST_HEAD(&still_in_list); + while (count < target && !list_empty(&dev_priv->mm.unbound_list)) { + obj = list_first_entry(&dev_priv->mm.unbound_list, + typeof(*obj), global_list); + list_move_tail(&obj->global_list, &still_in_list); + + if (!i915_gem_object_is_purgeable(obj) && purgeable_only) + continue; + + drm_gem_object_reference(&obj->base); + + if (i915_gem_object_put_pages(obj) == 0) + count += obj->base.size >> PAGE_SHIFT; + + drm_gem_object_unreference(&obj->base); + } + list_splice(&still_in_list, &dev_priv->mm.unbound_list); + + INIT_LIST_HEAD(&still_in_list); while (count < target && !list_empty(&dev_priv->mm.bound_list)) { struct i915_vma *vma, *v; obj = list_first_entry(&dev_priv->mm.bound_list, typeof(*obj), global_list); - list_move_tail(&obj->global_list, &still_bound_list); + list_move_tail(&obj->global_list, &still_in_list); if (!i915_gem_object_is_purgeable(obj) && purgeable_only) continue; - /* - * Hold a reference whilst we unbind this object, as we may - * end up waiting for and retiring requests. This might - * release the final reference (held by the active list) - * and result in the object being freed from under us. - * in this object being freed. - * - * Note 1: Shrinking the bound list is special since only active - * (and hence bound objects) can contain such limbo objects, so - * we don't need special tricks for shrinking the unbound list. - * The only other place where we have to be careful with active - * objects suddenly disappearing due to retiring requests is the - * eviction code. - * - * Note 2: Even though the bound list doesn't hold a reference - * to the object we can safely grab one here: The final object - * unreferencing and the bound_list are both protected by the - * dev->struct_mutex and so we won't ever be able to observe an - * object on the bound_list with a reference count equals 0. - */ drm_gem_object_reference(&obj->base); list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link) @@ -1822,7 +1982,7 @@ __i915_gem_shrink(struct drm_i915_private *dev_priv, long target, drm_gem_object_unreference(&obj->base); } - list_splice(&still_bound_list, &dev_priv->mm.bound_list); + list_splice(&still_in_list, &dev_priv->mm.bound_list); return count; } @@ -1836,17 +1996,8 @@ i915_gem_purge(struct drm_i915_private *dev_priv, long target) static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) { - struct drm_i915_gem_object *obj, *next; - long freed = 0; - i915_gem_evict_everything(dev_priv->dev); - - list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, - global_list) { - if (i915_gem_object_put_pages(obj) == 0) - freed += obj->base.size >> PAGE_SHIFT; - } - return freed; + return __i915_gem_shrink(dev_priv, LONG_MAX, false); } static int @@ -1950,7 +2101,19 @@ err_pages: page_cache_release(sg_page_iter_page(&sg_iter)); sg_free_table(st); kfree(st); - return PTR_ERR(page); + + /* shmemfs first checks if there is enough memory to allocate the page + * and reports ENOSPC should there be insufficient, along with the usual + * ENOMEM for a genuine allocation failure. + * + * We use ENOSPC in our driver to mean that we have run out of aperture + * space and so want to translate the error from shmemfs back to our + * usual understanding of ENOMEM. + */ + if (PTR_ERR(page) == -ENOSPC) + return -ENOMEM; + else + return PTR_ERR(page); } /* Ensure that the associated pages are gathered from the backing storage @@ -1971,8 +2134,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj) return 0; if (obj->madv != I915_MADV_WILLNEED) { - DRM_ERROR("Attempting to obtain a purgeable object\n"); - return -EINVAL; + DRM_DEBUG("Attempting to obtain a purgeable object\n"); + return -EFAULT; } BUG_ON(obj->pages_pin_count); @@ -1987,7 +2150,7 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj) static void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -2025,7 +2188,7 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, } void i915_vma_move_to_active(struct i915_vma *vma, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { list_move_tail(&vma->mm_list, &vma->vm->active_list); return i915_gem_object_move_to_active(vma->obj, ring); @@ -2035,13 +2198,17 @@ static void i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; - struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); + struct i915_address_space *vm; + struct i915_vma *vma; BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); BUG_ON(!obj->active); - list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list); + list_for_each_entry(vm, &dev_priv->vm_list, global_link) { + vma = i915_gem_obj_to_vma(obj, vm); + if (vma && !list_empty(&vma->mm_list)) + list_move_tail(&vma->mm_list, &vm->inactive_list); + } list_del_init(&obj->ring_list); obj->ring = NULL; @@ -2059,11 +2226,24 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) WARN_ON(i915_verify_lists(dev)); } +static void +i915_gem_object_retire(struct drm_i915_gem_object *obj) +{ + struct intel_engine_cs *ring = obj->ring; + + if (ring == NULL) + return; + + if (i915_seqno_passed(ring->get_seqno(ring, true), + obj->last_read_seqno)) + i915_gem_object_move_to_inactive(obj); +} + static int i915_gem_init_seqno(struct drm_device *dev, u32 seqno) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; int ret, i, j; /* Carefully retire all requests without writing to the rings */ @@ -2078,8 +2258,8 @@ i915_gem_init_seqno(struct drm_device *dev, u32 seqno) for_each_ring(ring, dev_priv, i) { intel_ring_init_seqno(ring, seqno); - for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) - ring->sync_seqno[j] = 0; + for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++) + ring->semaphore.sync_seqno[j] = 0; } return 0; @@ -2129,15 +2309,14 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) return 0; } -int __i915_add_request(struct intel_ring_buffer *ring, +int __i915_add_request(struct intel_engine_cs *ring, struct drm_file *file, struct drm_i915_gem_object *obj, u32 *out_seqno) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = ring->dev->dev_private; struct drm_i915_gem_request *request; u32 request_ring_position, request_start; - int was_empty; int ret; request_start = intel_ring_get_tail(ring); @@ -2188,7 +2367,6 @@ int __i915_add_request(struct intel_ring_buffer *ring, i915_gem_context_reference(request->ctx); request->emitted_jiffies = jiffies; - was_empty = list_empty(&ring->request_list); list_add_tail(&request->list, &ring->request_list); request->file_priv = NULL; @@ -2209,13 +2387,11 @@ int __i915_add_request(struct intel_ring_buffer *ring, if (!dev_priv->ums.mm_suspended) { i915_queue_hangcheck(ring->dev); - if (was_empty) { - cancel_delayed_work_sync(&dev_priv->mm.idle_work); - queue_delayed_work(dev_priv->wq, - &dev_priv->mm.retire_work, - round_jiffies_up_relative(HZ)); - intel_mark_busy(dev_priv->dev); - } + cancel_delayed_work_sync(&dev_priv->mm.idle_work); + queue_delayed_work(dev_priv->wq, + &dev_priv->mm.retire_work, + round_jiffies_up_relative(HZ)); + intel_mark_busy(dev_priv->dev); } if (out_seqno) @@ -2237,125 +2413,47 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) spin_unlock(&file_priv->mm.lock); } -static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj, - struct i915_address_space *vm) +static bool i915_context_is_banned(struct drm_i915_private *dev_priv, + const struct intel_context *ctx) { - if (acthd >= i915_gem_obj_offset(obj, vm) && - acthd < i915_gem_obj_offset(obj, vm) + obj->base.size) - return true; + unsigned long elapsed; - return false; -} + elapsed = get_seconds() - ctx->hang_stats.guilty_ts; -static bool i915_head_inside_request(const u32 acthd_unmasked, - const u32 request_start, - const u32 request_end) -{ - const u32 acthd = acthd_unmasked & HEAD_ADDR; + if (ctx->hang_stats.banned) + return true; - if (request_start < request_end) { - if (acthd >= request_start && acthd < request_end) + if (elapsed <= DRM_I915_CTX_BAN_PERIOD) { + if (!i915_gem_context_is_default(ctx)) { + DRM_DEBUG("context hanging too fast, banning!\n"); return true; - } else if (request_start > request_end) { - if (acthd >= request_start || acthd < request_end) - return true; - } - - return false; -} - -static struct i915_address_space * -request_to_vm(struct drm_i915_gem_request *request) -{ - struct drm_i915_private *dev_priv = request->ring->dev->dev_private; - struct i915_address_space *vm; - - vm = &dev_priv->gtt.base; - - return vm; -} - -static bool i915_request_guilty(struct drm_i915_gem_request *request, - const u32 acthd, bool *inside) -{ - /* There is a possibility that unmasked head address - * pointing inside the ring, matches the batch_obj address range. - * However this is extremely unlikely. - */ - if (request->batch_obj) { - if (i915_head_inside_object(acthd, request->batch_obj, - request_to_vm(request))) { - *inside = true; + } else if (i915_stop_ring_allow_ban(dev_priv)) { + if (i915_stop_ring_allow_warn(dev_priv)) + DRM_ERROR("gpu hanging too fast, banning!\n"); return true; } } - if (i915_head_inside_request(acthd, request->head, request->tail)) { - *inside = false; - return true; - } - - return false; -} - -static bool i915_context_is_banned(const struct i915_ctx_hang_stats *hs) -{ - const unsigned long elapsed = get_seconds() - hs->guilty_ts; - - if (hs->banned) - return true; - - if (elapsed <= DRM_I915_CTX_BAN_PERIOD) { - DRM_ERROR("context hanging too fast, declaring banned!\n"); - return true; - } - return false; } -static void i915_set_reset_status(struct intel_ring_buffer *ring, - struct drm_i915_gem_request *request, - u32 acthd) +static void i915_set_reset_status(struct drm_i915_private *dev_priv, + struct intel_context *ctx, + const bool guilty) { - struct i915_ctx_hang_stats *hs = NULL; - bool inside, guilty; - unsigned long offset = 0; - - /* Innocent until proven guilty */ - guilty = false; - - if (request->batch_obj) - offset = i915_gem_obj_offset(request->batch_obj, - request_to_vm(request)); + struct i915_ctx_hang_stats *hs; - if (ring->hangcheck.action != HANGCHECK_WAIT && - i915_request_guilty(request, acthd, &inside)) { - DRM_DEBUG("%s hung %s bo (0x%lx ctx %d) at 0x%x\n", - ring->name, - inside ? "inside" : "flushing", - offset, - request->ctx ? request->ctx->id : 0, - acthd); + if (WARN_ON(!ctx)) + return; - guilty = true; - } + hs = &ctx->hang_stats; - /* If contexts are disabled or this is the default context, use - * file_priv->reset_state - */ - if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID) - hs = &request->ctx->hang_stats; - else if (request->file_priv) - hs = &request->file_priv->hang_stats; - - if (hs) { - if (guilty) { - hs->banned = i915_context_is_banned(hs); - hs->batch_active++; - hs->guilty_ts = get_seconds(); - } else { - hs->batch_pending++; - } + if (guilty) { + hs->banned = i915_context_is_banned(dev_priv, ctx); + hs->batch_active++; + hs->guilty_ts = get_seconds(); + } else { + hs->batch_pending++; } } @@ -2370,23 +2468,45 @@ static void i915_gem_free_request(struct drm_i915_gem_request *request) kfree(request); } -static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv, - struct intel_ring_buffer *ring) +struct drm_i915_gem_request * +i915_gem_find_active_request(struct intel_engine_cs *ring) { - u32 completed_seqno = ring->get_seqno(ring, false); - u32 acthd = intel_ring_get_active_head(ring); struct drm_i915_gem_request *request; + u32 completed_seqno; + + completed_seqno = ring->get_seqno(ring, false); list_for_each_entry(request, &ring->request_list, list) { if (i915_seqno_passed(completed_seqno, request->seqno)) continue; - i915_set_reset_status(ring, request, acthd); + return request; } + + return NULL; +} + +static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv, + struct intel_engine_cs *ring) +{ + struct drm_i915_gem_request *request; + bool ring_hung; + + request = i915_gem_find_active_request(ring); + + if (request == NULL) + return; + + ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG; + + i915_set_reset_status(dev_priv, request->ctx, ring_hung); + + list_for_each_entry_continue(request, &ring->request_list, list) + i915_set_reset_status(dev_priv, request->ctx, false); } static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj; @@ -2414,6 +2534,11 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, i915_gem_free_request(request); } + + /* These may not have been flush before the reset, do so now */ + kfree(ring->preallocated_lazy_request); + ring->preallocated_lazy_request = NULL; + ring->outstanding_lazy_seqno = 0; } void i915_gem_restore_fences(struct drm_device *dev) @@ -2440,7 +2565,7 @@ void i915_gem_restore_fences(struct drm_device *dev) void i915_gem_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; int i; /* @@ -2454,7 +2579,7 @@ void i915_gem_reset(struct drm_device *dev) for_each_ring(ring, dev_priv, i) i915_gem_reset_ring_cleanup(dev_priv, ring); - i915_gem_cleanup_ringbuffer(dev); + i915_gem_context_reset(dev); i915_gem_restore_fences(dev); } @@ -2463,7 +2588,7 @@ void i915_gem_reset(struct drm_device *dev) * This function clears the request list as sequence numbers are passed. */ void -i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) +i915_gem_retire_requests_ring(struct intel_engine_cs *ring) { uint32_t seqno; @@ -2474,6 +2599,24 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) seqno = ring->get_seqno(ring, true); + /* Move any buffers on the active list that are no longer referenced + * by the ringbuffer to the flushing/inactive lists as appropriate, + * before we free the context associated with the requests. + */ + while (!list_empty(&ring->active_list)) { + struct drm_i915_gem_object *obj; + + obj = list_first_entry(&ring->active_list, + struct drm_i915_gem_object, + ring_list); + + if (!i915_seqno_passed(seqno, obj->last_read_seqno)) + break; + + i915_gem_object_move_to_inactive(obj); + } + + while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; @@ -2490,27 +2633,11 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) * of tail of the request to update the last known position * of the GPU head. */ - ring->last_retired_head = request->tail; + ring->buffer->last_retired_head = request->tail; i915_gem_free_request(request); } - /* Move any buffers on the active list that are no longer referenced - * by the ringbuffer to the flushing/inactive lists as appropriate. - */ - while (!list_empty(&ring->active_list)) { - struct drm_i915_gem_object *obj; - - obj = list_first_entry(&ring->active_list, - struct drm_i915_gem_object, - ring_list); - - if (!i915_seqno_passed(seqno, obj->last_read_seqno)) - break; - - i915_gem_object_move_to_inactive(obj); - } - if (unlikely(ring->trace_irq_seqno && i915_seqno_passed(seqno, ring->trace_irq_seqno))) { ring->irq_put(ring); @@ -2523,8 +2650,8 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) bool i915_gem_retire_requests(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; bool idle = true; int i; @@ -2615,10 +2742,10 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj) int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_wait *args = data; struct drm_i915_gem_object *obj; - struct intel_ring_buffer *ring = NULL; + struct intel_engine_cs *ring = NULL; struct timespec timeout_stack, *timeout = NULL; unsigned reset_counter; u32 seqno = 0; @@ -2689,9 +2816,9 @@ out: */ int i915_gem_object_sync(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *to) + struct intel_engine_cs *to) { - struct intel_ring_buffer *from = obj->ring; + struct intel_engine_cs *from = obj->ring; u32 seqno; int ret, idx; @@ -2704,7 +2831,7 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj, idx = intel_ring_sync_index(from, to); seqno = obj->last_read_seqno; - if (seqno <= from->sync_seqno[idx]) + if (seqno <= from->semaphore.sync_seqno[idx]) return 0; ret = i915_gem_check_olr(obj->ring, seqno); @@ -2712,13 +2839,13 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj, return ret; trace_i915_gem_ring_sync_to(from, to, seqno); - ret = to->sync_to(to, from, seqno); + ret = to->semaphore.sync_to(to, from, seqno); if (!ret) /* We use last_read_seqno because sync_to() * might have just caused seqno wrap under * the radar. */ - from->sync_seqno[idx] = obj->last_read_seqno; + from->semaphore.sync_seqno[idx] = obj->last_read_seqno; return ret; } @@ -2750,22 +2877,18 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) int i915_vma_unbind(struct i915_vma *vma) { struct drm_i915_gem_object *obj = vma->obj; - drm_i915_private_t *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; int ret; - /* For now we only ever use 1 vma per object */ - WARN_ON(!list_is_singular(&obj->vma_list)); - if (list_empty(&vma->vma_link)) return 0; if (!drm_mm_node_allocated(&vma->node)) { i915_gem_vma_destroy(vma); - return 0; } - if (obj->pin_count) + if (vma->pin_count) return -EBUSY; BUG_ON(obj->pages == NULL); @@ -2778,24 +2901,22 @@ int i915_vma_unbind(struct i915_vma *vma) * cause memory corruption through use-after-free. */ - i915_gem_object_finish_gtt(obj); + if (i915_is_ggtt(vma->vm)) { + i915_gem_object_finish_gtt(obj); - /* release the fence reg _after_ flushing */ - ret = i915_gem_object_put_fence(obj); - if (ret) - return ret; + /* release the fence reg _after_ flushing */ + ret = i915_gem_object_put_fence(obj); + if (ret) + return ret; + } trace_i915_vma_unbind(vma); - if (obj->has_global_gtt_mapping) - i915_gem_gtt_unbind_object(obj); - if (obj->has_aliasing_ppgtt_mapping) { - i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); - obj->has_aliasing_ppgtt_mapping = 0; - } + vma->unbind_vma(vma); + i915_gem_gtt_finish_object(obj); - list_del(&vma->mm_list); + list_del_init(&vma->mm_list); /* Avoid an unnecessary call to unbind on rebind. */ if (i915_is_ggtt(vma->vm)) obj->map_and_fenceable = true; @@ -2817,35 +2938,15 @@ int i915_vma_unbind(struct i915_vma *vma) return 0; } -/** - * Unbinds an object from the global GTT aperture. - */ -int -i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - struct i915_address_space *ggtt = &dev_priv->gtt.base; - - if (!i915_gem_obj_ggtt_bound(obj)) - return 0; - - if (obj->pin_count) - return -EBUSY; - - BUG_ON(obj->pages == NULL); - - return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt)); -} - int i915_gpu_idle(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; int ret, i; /* Flush everything onto the inactive list. */ for_each_ring(ring, dev_priv, i) { - ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); + ret = i915_switch_context(ring, ring->default_context); if (ret) return ret; @@ -2860,7 +2961,7 @@ int i915_gpu_idle(struct drm_device *dev) static void i965_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int fence_reg; int fence_pitch_shift; @@ -2912,7 +3013,7 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg, static void i915_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 val; if (obj) { @@ -2956,7 +3057,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg, static void i830_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; uint32_t val; if (obj) { @@ -3081,6 +3182,9 @@ i915_gem_object_put_fence(struct drm_i915_gem_object *obj) fence = &dev_priv->fence_regs[obj->fence_reg]; + if (WARN_ON(fence->pin_count)) + return -EBUSY; + i915_gem_object_fence_lost(obj); i915_gem_object_update_fence(obj, fence, false); @@ -3259,18 +3363,19 @@ static void i915_gem_verify_gtt(struct drm_device *dev) /** * Finds free space in the GTT aperture and binds the object there. */ -static int +static struct i915_vma * i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, struct i915_address_space *vm, unsigned alignment, - bool map_and_fenceable, - bool nonblocking) + uint64_t flags) { struct drm_device *dev = obj->base.dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 size, fence_size, fence_alignment, unfenced_alignment; - size_t gtt_max = - map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total; + unsigned long start = + flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; + unsigned long end = + flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total; struct i915_vma *vma; int ret; @@ -3282,57 +3387,52 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, obj->tiling_mode, true); unfenced_alignment = i915_gem_get_gtt_alignment(dev, - obj->base.size, - obj->tiling_mode, false); + obj->base.size, + obj->tiling_mode, false); if (alignment == 0) - alignment = map_and_fenceable ? fence_alignment : + alignment = flags & PIN_MAPPABLE ? fence_alignment : unfenced_alignment; - if (map_and_fenceable && alignment & (fence_alignment - 1)) { - DRM_ERROR("Invalid object alignment requested %u\n", alignment); - return -EINVAL; + if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) { + DRM_DEBUG("Invalid object alignment requested %u\n", alignment); + return ERR_PTR(-EINVAL); } - size = map_and_fenceable ? fence_size : obj->base.size; + size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; /* If the object is bigger than the entire aperture, reject it early * before evicting everything in a vain attempt to find space. */ - if (obj->base.size > gtt_max) { - DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n", + if (obj->base.size > end) { + DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n", obj->base.size, - map_and_fenceable ? "mappable" : "total", - gtt_max); - return -E2BIG; + flags & PIN_MAPPABLE ? "mappable" : "total", + end); + return ERR_PTR(-E2BIG); } ret = i915_gem_object_get_pages(obj); if (ret) - return ret; + return ERR_PTR(ret); i915_gem_object_pin_pages(obj); - BUG_ON(!i915_is_ggtt(vm)); - vma = i915_gem_obj_lookup_or_create_vma(obj, vm); - if (IS_ERR(vma)) { - ret = PTR_ERR(vma); + if (IS_ERR(vma)) goto err_unpin; - } - - /* For now we only ever use 1 vma per object */ - WARN_ON(!list_is_singular(&obj->vma_list)); search_free: ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, size, alignment, - obj->cache_level, 0, gtt_max, - DRM_MM_SEARCH_DEFAULT); + obj->cache_level, + start, end, + DRM_MM_SEARCH_DEFAULT, + DRM_MM_CREATE_DEFAULT); if (ret) { ret = i915_gem_evict_something(dev, vm, size, alignment, obj->cache_level, - map_and_fenceable, - nonblocking); + start, end, + flags); if (ret == 0) goto search_free; @@ -3363,19 +3463,23 @@ search_free: obj->map_and_fenceable = mappable && fenceable; } - WARN_ON(map_and_fenceable && !obj->map_and_fenceable); + WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); + + trace_i915_vma_bind(vma, flags); + vma->bind_vma(vma, obj->cache_level, + flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0); - trace_i915_vma_bind(vma, map_and_fenceable); i915_gem_verify_gtt(dev); - return 0; + return vma; err_remove_node: drm_mm_remove_node(&vma->node); err_free_vma: i915_gem_vma_destroy(vma); + vma = ERR_PTR(ret); err_unpin: i915_gem_object_unpin_pages(obj); - return ret; + return vma; } bool @@ -3470,7 +3574,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) { - drm_i915_private_t *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; uint32_t old_write_domain, old_read_domains; int ret; @@ -3485,6 +3589,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) if (ret) return ret; + i915_gem_object_retire(obj); i915_gem_object_flush_cpu_write_domain(obj, false); /* Serialise direct access to this object with the barriers for @@ -3528,25 +3633,22 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct drm_device *dev = obj->base.dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct i915_vma *vma; + struct i915_vma *vma, *next; int ret; if (obj->cache_level == cache_level) return 0; - if (obj->pin_count) { + if (i915_gem_obj_is_pinned(obj)) { DRM_DEBUG("can not change the cache level of pinned objects\n"); return -EBUSY; } - list_for_each_entry(vma, &obj->vma_list, vma_link) { + list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) { ret = i915_vma_unbind(vma); if (ret) return ret; - - break; } } @@ -3567,11 +3669,10 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, return ret; } - if (obj->has_global_gtt_mapping) - i915_gem_gtt_bind_object(obj, cache_level); - if (obj->has_aliasing_ppgtt_mapping) - i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, - obj, cache_level); + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (drm_mm_node_allocated(&vma->node)) + vma->bind_vma(vma, cache_level, + obj->has_global_gtt_mapping ? GLOBAL_BIND : 0); } list_for_each_entry(vma, &obj->vma_list, vma_link) @@ -3587,6 +3688,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, * in obj->write_domain and have been skipping the clflushes. * Just set it to the CPU cache for now. */ + i915_gem_object_retire(obj); WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); old_read_domains = obj->base.read_domains; @@ -3684,6 +3786,15 @@ unlock: static bool is_pin_display(struct drm_i915_gem_object *obj) { + struct i915_vma *vma; + + if (list_empty(&obj->vma_list)) + return false; + + vma = i915_gem_obj_to_ggtt(obj); + if (!vma) + return false; + /* There are 3 sources that pin objects: * 1. The display engine (scanouts, sprites, cursors); * 2. Reservations for execbuffer; @@ -3695,7 +3806,7 @@ static bool is_pin_display(struct drm_i915_gem_object *obj) * subtracting the potential reference by the user, any pin_count * remains, it must be due to another use by the display engine. */ - return obj->pin_count - !!obj->user_pin_count; + return vma->pin_count - !!obj->user_pin_count; } /* @@ -3706,9 +3817,10 @@ static bool is_pin_display(struct drm_i915_gem_object *obj) int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, - struct intel_ring_buffer *pipelined) + struct intel_engine_cs *pipelined) { u32 old_read_domains, old_write_domain; + bool was_pin_display; int ret; if (pipelined != obj->ring) { @@ -3720,6 +3832,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, /* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. */ + was_pin_display = obj->pin_display; obj->pin_display = true; /* The display engine is not coherent with the LLC cache on gen6. As @@ -3740,7 +3853,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, * (e.g. libkms for the bootup splash), we have to ensure that we * always use map_and_fenceable for all scanout buffers. */ - ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false); + ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE); if (ret) goto err_unpin_display; @@ -3762,14 +3875,15 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, return 0; err_unpin_display: - obj->pin_display = is_pin_display(obj); + WARN_ON(was_pin_display != is_pin_display(obj)); + obj->pin_display = was_pin_display; return ret; } void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) { - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); obj->pin_display = is_pin_display(obj); } @@ -3809,6 +3923,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) if (ret) return ret; + i915_gem_object_retire(obj); i915_gem_object_flush_gtt_write_domain(obj); old_write_domain = obj->base.write_domain; @@ -3858,7 +3973,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) struct drm_i915_file_private *file_priv = file->driver_priv; unsigned long recent_enough = jiffies - msecs_to_jiffies(20); struct drm_i915_gem_request *request; - struct intel_ring_buffer *ring = NULL; + struct intel_engine_cs *ring = NULL; unsigned reset_counter; u32 seqno = 0; int ret; @@ -3892,72 +4007,117 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) return ret; } +static bool +i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags) +{ + struct drm_i915_gem_object *obj = vma->obj; + + if (alignment && + vma->node.start & (alignment - 1)) + return true; + + if (flags & PIN_MAPPABLE && !obj->map_and_fenceable) + return true; + + if (flags & PIN_OFFSET_BIAS && + vma->node.start < (flags & PIN_OFFSET_MASK)) + return true; + + return false; +} + int i915_gem_object_pin(struct drm_i915_gem_object *obj, struct i915_address_space *vm, uint32_t alignment, - bool map_and_fenceable, - bool nonblocking) + uint64_t flags) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; struct i915_vma *vma; int ret; - if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) - return -EBUSY; + if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base)) + return -ENODEV; - WARN_ON(map_and_fenceable && !i915_is_ggtt(vm)); + if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm))) + return -EINVAL; vma = i915_gem_obj_to_vma(obj, vm); - if (vma) { - if ((alignment && - vma->node.start & (alignment - 1)) || - (map_and_fenceable && !obj->map_and_fenceable)) { - WARN(obj->pin_count, + if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) + return -EBUSY; + + if (i915_vma_misplaced(vma, alignment, flags)) { + WARN(vma->pin_count, "bo is already pinned with incorrect alignment:" " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d," " obj->map_and_fenceable=%d\n", i915_gem_obj_offset(obj, vm), alignment, - map_and_fenceable, + !!(flags & PIN_MAPPABLE), obj->map_and_fenceable); ret = i915_vma_unbind(vma); if (ret) return ret; + + vma = NULL; } } - if (!i915_gem_obj_bound(obj, vm)) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; - - ret = i915_gem_object_bind_to_vm(obj, vm, alignment, - map_and_fenceable, - nonblocking); - if (ret) - return ret; - - if (!dev_priv->mm.aliasing_ppgtt) - i915_gem_gtt_bind_object(obj, obj->cache_level); + if (vma == NULL || !drm_mm_node_allocated(&vma->node)) { + vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags); + if (IS_ERR(vma)) + return PTR_ERR(vma); } - if (!obj->has_global_gtt_mapping && map_and_fenceable) - i915_gem_gtt_bind_object(obj, obj->cache_level); + if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping) + vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND); - obj->pin_count++; - obj->pin_mappable |= map_and_fenceable; + vma->pin_count++; + if (flags & PIN_MAPPABLE) + obj->pin_mappable |= true; return 0; } void -i915_gem_object_unpin(struct drm_i915_gem_object *obj) +i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj) { - BUG_ON(obj->pin_count == 0); - BUG_ON(!i915_gem_obj_bound_any(obj)); + struct i915_vma *vma = i915_gem_obj_to_ggtt(obj); + + BUG_ON(!vma); + BUG_ON(vma->pin_count == 0); + BUG_ON(!i915_gem_obj_ggtt_bound(obj)); - if (--obj->pin_count == 0) + if (--vma->pin_count == 0) obj->pin_mappable = false; } +bool +i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) +{ + if (obj->fence_reg != I915_FENCE_REG_NONE) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj); + + WARN_ON(!ggtt_vma || + dev_priv->fence_regs[obj->fence_reg].pin_count > + ggtt_vma->pin_count); + dev_priv->fence_regs[obj->fence_reg].pin_count++; + return true; + } else + return false; +} + +void +i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) +{ + if (obj->fence_reg != I915_FENCE_REG_NONE) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); + dev_priv->fence_regs[obj->fence_reg].pin_count--; + } +} + int i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_file *file) @@ -3966,6 +4126,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int ret; + if (INTEL_INFO(dev)->gen >= 6) + return -ENODEV; + ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; @@ -3977,13 +4140,13 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, } if (obj->madv != I915_MADV_WILLNEED) { - DRM_ERROR("Attempting to pin a purgeable buffer\n"); - ret = -EINVAL; + DRM_DEBUG("Attempting to pin a purgeable buffer\n"); + ret = -EFAULT; goto out; } if (obj->pin_filp != NULL && obj->pin_filp != file) { - DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", + DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; @@ -3995,7 +4158,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, } if (obj->user_pin_count == 0) { - ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false); + ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE); if (ret) goto out; } @@ -4030,7 +4193,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data, } if (obj->pin_filp != file) { - DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", + DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; @@ -4038,7 +4201,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data, obj->user_pin_count--; if (obj->user_pin_count == 0) { obj->pin_filp = NULL; - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); } out: @@ -4118,7 +4281,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, goto unlock; } - if (obj->pin_count) { + if (i915_gem_obj_is_pinned(obj)) { ret = -EINVAL; goto out; } @@ -4215,26 +4378,46 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, return obj; } +static bool discard_backing_storage(struct drm_i915_gem_object *obj) +{ + /* If we are the last user of the backing storage (be it shmemfs + * pages or stolen etc), we know that the pages are going to be + * immediately released. In this case, we can then skip copying + * back the contents from the GPU. + */ + + if (obj->madv != I915_MADV_WILLNEED) + return false; + + if (obj->base.filp == NULL) + return true; + + /* At first glance, this looks racy, but then again so would be + * userspace racing mmap against close. However, the first external + * reference to the filp can only be obtained through the + * i915_gem_mmap_ioctl() which safeguards us against the user + * acquiring such a reference whilst we are in the middle of + * freeing the object. + */ + return atomic_long_read(&obj->base.filp->f_count) == 1; +} + void i915_gem_free_object(struct drm_gem_object *gem_obj) { struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); struct drm_device *dev = obj->base.dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct i915_vma *vma, *next; intel_runtime_pm_get(dev_priv); trace_i915_gem_object_destroy(obj); - if (obj->phys_obj) - i915_gem_detach_phys_object(dev, obj); - - obj->pin_count = 0; - /* NB: 0 or 1 elements */ - WARN_ON(!list_empty(&obj->vma_list) && - !list_is_singular(&obj->vma_list)); list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { - int ret = i915_vma_unbind(vma); + int ret; + + vma->pin_count = 0; + ret = i915_vma_unbind(vma); if (WARN_ON(ret == -ERESTARTSYS)) { bool was_interruptible; @@ -4247,6 +4430,8 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) } } + i915_gem_object_detach_phys(obj); + /* Stolen objects don't hold a ref, but do hold pin count. Fix that up * before progressing. */ if (obj->stolen) @@ -4254,6 +4439,8 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) if (WARN_ON(obj->pages_pin_count)) obj->pages_pin_count = 0; + if (discard_backing_storage(obj)) + obj->madv = I915_MADV_DONTNEED; i915_gem_object_put_pages(obj); i915_gem_object_free_mmap_offset(obj); i915_gem_object_release_stolen(obj); @@ -4263,6 +4450,9 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) if (obj->base.import_attach) drm_prime_gem_destroy(&obj->base, NULL); + if (obj->ops->release) + obj->ops->release(obj); + drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev_priv, obj->base.size); @@ -4283,41 +4473,6 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, return NULL; } -static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj, - struct i915_address_space *vm) -{ - struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); - if (vma == NULL) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&vma->vma_link); - INIT_LIST_HEAD(&vma->mm_list); - INIT_LIST_HEAD(&vma->exec_list); - vma->vm = vm; - vma->obj = obj; - - /* Keep GGTT vmas first to make debug easier */ - if (i915_is_ggtt(vm)) - list_add(&vma->vma_link, &obj->vma_list); - else - list_add_tail(&vma->vma_link, &obj->vma_list); - - return vma; -} - -struct i915_vma * -i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, - struct i915_address_space *vm) -{ - struct i915_vma *vma; - - vma = i915_gem_obj_to_vma(obj, vm); - if (!vma) - vma = __i915_gem_vma_create(obj, vm); - - return vma; -} - void i915_gem_vma_destroy(struct i915_vma *vma) { WARN_ON(vma->node.allocated); @@ -4331,10 +4486,21 @@ void i915_gem_vma_destroy(struct i915_vma *vma) kfree(vma); } +static void +i915_gem_stop_ringbuffers(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; + int i; + + for_each_ring(ring, dev_priv, i) + intel_stop_ring_buffer(ring); +} + int i915_gem_suspend(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret = 0; mutex_lock(&dev->struct_mutex); @@ -4352,7 +4518,7 @@ i915_gem_suspend(struct drm_device *dev) i915_gem_evict_everything(dev); i915_kernel_lost_context(dev); - i915_gem_cleanup_ringbuffer(dev); + i915_gem_stop_ringbuffers(dev); /* Hack! Don't let anybody do execbuf while we don't control the chip. * We need to replace this with a semaphore, or something. @@ -4373,10 +4539,10 @@ err: return ret; } -int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice) +int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; int i, ret; @@ -4406,7 +4572,7 @@ int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice) void i915_gem_init_swizzling(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen < 5 || dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) @@ -4472,13 +4638,20 @@ static int i915_gem_init_rings(struct drm_device *dev) goto cleanup_blt_ring; } + if (HAS_BSD2(dev)) { + ret = intel_init_bsd2_ring_buffer(dev); + if (ret) + goto cleanup_vebox_ring; + } ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); if (ret) - goto cleanup_vebox_ring; + goto cleanup_bsd2_ring; return 0; +cleanup_bsd2_ring: + intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]); cleanup_vebox_ring: intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); cleanup_blt_ring: @@ -4494,7 +4667,7 @@ cleanup_render_ring: int i915_gem_init_hw(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret, i; if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) @@ -4508,9 +4681,15 @@ i915_gem_init_hw(struct drm_device *dev) LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); if (HAS_PCH_NOP(dev)) { - u32 temp = I915_READ(GEN7_MSG_CTL); - temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); - I915_WRITE(GEN7_MSG_CTL, temp); + if (IS_IVYBRIDGE(dev)) { + u32 temp = I915_READ(GEN7_MSG_CTL); + temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); + I915_WRITE(GEN7_MSG_CTL, temp); + } else if (INTEL_INFO(dev)->gen >= 7) { + u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); + temp &= ~RESET_PCH_HANDSHAKE_ENABLE; + I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); + } } i915_gem_init_swizzling(dev); @@ -4523,25 +4702,19 @@ i915_gem_init_hw(struct drm_device *dev) i915_gem_l3_remap(&dev_priv->ring[RCS], i); /* - * XXX: There was some w/a described somewhere suggesting loading - * contexts before PPGTT. + * XXX: Contexts should only be initialized once. Doing a switch to the + * default context switch however is something we'd like to do after + * reset or thaw (the latter may not actually be necessary for HW, but + * goes with our code better). Context switching requires rings (for + * the do_switch), but before enabling PPGTT. So don't move this. */ - ret = i915_gem_context_init(dev); - if (ret) { + ret = i915_gem_context_enable(dev_priv); + if (ret && ret != -EIO) { + DRM_ERROR("Context enable failed %d\n", ret); i915_gem_cleanup_ringbuffer(dev); - DRM_ERROR("Context initialization failed %d\n", ret); - return ret; - } - - if (dev_priv->mm.aliasing_ppgtt) { - ret = dev_priv->mm.aliasing_ppgtt->enable(dev); - if (ret) { - i915_gem_cleanup_aliasing_ppgtt(dev); - DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n"); - } } - return 0; + return ret; } int i915_gem_init(struct drm_device *dev) @@ -4553,31 +4726,44 @@ int i915_gem_init(struct drm_device *dev) if (IS_VALLEYVIEW(dev)) { /* VLVA0 (potential hack), BIOS isn't actually waking us */ - I915_WRITE(VLV_GTLC_WAKE_CTRL, 1); - if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10)) + I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ); + if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & + VLV_GTLC_ALLOWWAKEACK), 10)) DRM_DEBUG_DRIVER("allow wake ack timed out\n"); } + i915_gem_init_userptr(dev); i915_gem_init_global_gtt(dev); - ret = i915_gem_init_hw(dev); - mutex_unlock(&dev->struct_mutex); + ret = i915_gem_context_init(dev); if (ret) { - i915_gem_cleanup_aliasing_ppgtt(dev); + mutex_unlock(&dev->struct_mutex); return ret; } + ret = i915_gem_init_hw(dev); + if (ret == -EIO) { + /* Allow ring initialisation to fail by marking the GPU as + * wedged. But we only want to do this where the GPU is angry, + * for all other failure, such as an allocation failure, bail. + */ + DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); + atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + ret = 0; + } + mutex_unlock(&dev->struct_mutex); + /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) dev_priv->dri1.allow_batchbuffer = 1; - return 0; + return ret; } void i915_gem_cleanup_ringbuffer(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; int i; for_each_ring(ring, dev_priv, i) @@ -4609,16 +4795,15 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, } BUG_ON(!list_empty(&dev_priv->gtt.base.active_list)); - mutex_unlock(&dev->struct_mutex); - ret = drm_irq_install(dev); + ret = drm_irq_install(dev, dev->pdev->irq); if (ret) goto cleanup_ringbuffer; + mutex_unlock(&dev->struct_mutex); return 0; cleanup_ringbuffer: - mutex_lock(&dev->struct_mutex); i915_gem_cleanup_ringbuffer(dev); dev_priv->ums.mm_suspended = 1; mutex_unlock(&dev->struct_mutex); @@ -4633,7 +4818,9 @@ i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; + mutex_lock(&dev->struct_mutex); drm_irq_uninstall(dev); + mutex_unlock(&dev->struct_mutex); return i915_gem_suspend(dev); } @@ -4652,26 +4839,28 @@ i915_gem_lastclose(struct drm_device *dev) } static void -init_ring_lists(struct intel_ring_buffer *ring) +init_ring_lists(struct intel_engine_cs *ring) { INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); } -static void i915_init_vm(struct drm_i915_private *dev_priv, - struct i915_address_space *vm) +void i915_init_vm(struct drm_i915_private *dev_priv, + struct i915_address_space *vm) { + if (!i915_is_ggtt(vm)) + drm_mm_init(&vm->mm, vm->start, vm->total); vm->dev = dev_priv->dev; INIT_LIST_HEAD(&vm->active_list); INIT_LIST_HEAD(&vm->inactive_list); INIT_LIST_HEAD(&vm->global_link); - list_add(&vm->global_link, &dev_priv->vm_list); + list_add_tail(&vm->global_link, &dev_priv->vm_list); } void i915_gem_load(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int i; dev_priv->slab = @@ -4698,7 +4887,7 @@ i915_gem_load(struct drm_device *dev) init_waitqueue_head(&dev_priv->gpu_error.reset_queue); /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ - if (IS_GEN3(dev)) { + if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) { I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); } @@ -4725,194 +4914,13 @@ i915_gem_load(struct drm_device *dev) dev_priv->mm.interruptible = true; - dev_priv->mm.inactive_shrinker.scan_objects = i915_gem_inactive_scan; - dev_priv->mm.inactive_shrinker.count_objects = i915_gem_inactive_count; - dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS; - register_shrinker(&dev_priv->mm.inactive_shrinker); -} - -/* - * Create a physically contiguous memory object for this object - * e.g. for cursor + overlay regs - */ -static int i915_gem_init_phys_object(struct drm_device *dev, - int id, int size, int align) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_phys_object *phys_obj; - int ret; - - if (dev_priv->mm.phys_objs[id - 1] || !size) - return 0; + dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan; + dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count; + dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&dev_priv->mm.shrinker); - phys_obj = kzalloc(sizeof(*phys_obj), GFP_KERNEL); - if (!phys_obj) - return -ENOMEM; - - phys_obj->id = id; - - phys_obj->handle = drm_pci_alloc(dev, size, align); - if (!phys_obj->handle) { - ret = -ENOMEM; - goto kfree_obj; - } -#ifdef CONFIG_X86 - set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); -#endif - - dev_priv->mm.phys_objs[id - 1] = phys_obj; - - return 0; -kfree_obj: - kfree(phys_obj); - return ret; -} - -static void i915_gem_free_phys_object(struct drm_device *dev, int id) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_phys_object *phys_obj; - - if (!dev_priv->mm.phys_objs[id - 1]) - return; - - phys_obj = dev_priv->mm.phys_objs[id - 1]; - if (phys_obj->cur_obj) { - i915_gem_detach_phys_object(dev, phys_obj->cur_obj); - } - -#ifdef CONFIG_X86 - set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); -#endif - drm_pci_free(dev, phys_obj->handle); - kfree(phys_obj); - dev_priv->mm.phys_objs[id - 1] = NULL; -} - -void i915_gem_free_all_phys_object(struct drm_device *dev) -{ - int i; - - for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) - i915_gem_free_phys_object(dev, i); -} - -void i915_gem_detach_phys_object(struct drm_device *dev, - struct drm_i915_gem_object *obj) -{ - struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; - char *vaddr; - int i; - int page_count; - - if (!obj->phys_obj) - return; - vaddr = obj->phys_obj->handle->vaddr; - - page_count = obj->base.size / PAGE_SIZE; - for (i = 0; i < page_count; i++) { - struct page *page = shmem_read_mapping_page(mapping, i); - if (!IS_ERR(page)) { - char *dst = kmap_atomic(page); - memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE); - kunmap_atomic(dst); - - drm_clflush_pages(&page, 1); - - set_page_dirty(page); - mark_page_accessed(page); - page_cache_release(page); - } - } - i915_gem_chipset_flush(dev); - - obj->phys_obj->cur_obj = NULL; - obj->phys_obj = NULL; -} - -int -i915_gem_attach_phys_object(struct drm_device *dev, - struct drm_i915_gem_object *obj, - int id, - int align) -{ - struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; - drm_i915_private_t *dev_priv = dev->dev_private; - int ret = 0; - int page_count; - int i; - - if (id > I915_MAX_PHYS_OBJECT) - return -EINVAL; - - if (obj->phys_obj) { - if (obj->phys_obj->id == id) - return 0; - i915_gem_detach_phys_object(dev, obj); - } - - /* create a new object */ - if (!dev_priv->mm.phys_objs[id - 1]) { - ret = i915_gem_init_phys_object(dev, id, - obj->base.size, align); - if (ret) { - DRM_ERROR("failed to init phys object %d size: %zu\n", - id, obj->base.size); - return ret; - } - } - - /* bind to the object */ - obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; - obj->phys_obj->cur_obj = obj; - - page_count = obj->base.size / PAGE_SIZE; - - for (i = 0; i < page_count; i++) { - struct page *page; - char *dst, *src; - - page = shmem_read_mapping_page(mapping, i); - if (IS_ERR(page)) - return PTR_ERR(page); - - src = kmap_atomic(page); - dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE); - memcpy(dst, src, PAGE_SIZE); - kunmap_atomic(src); - - mark_page_accessed(page); - page_cache_release(page); - } - - return 0; -} - -static int -i915_gem_phys_pwrite(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) -{ - void *vaddr = obj->phys_obj->handle->vaddr + args->offset; - char __user *user_data = to_user_ptr(args->data_ptr); - - if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { - unsigned long unwritten; - - /* The physical object once assigned is fixed for the lifetime - * of the obj, so we can safely drop the lock and continue - * to access vaddr. - */ - mutex_unlock(&dev->struct_mutex); - unwritten = copy_from_user(vaddr, user_data, args->size); - mutex_lock(&dev->struct_mutex); - if (unwritten) - return -EFAULT; - } - - i915_gem_chipset_flush(dev); - return 0; + dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; + register_oom_notifier(&dev_priv->mm.oom_notifier); } void i915_gem_release(struct drm_device *dev, struct drm_file *file) @@ -4950,6 +4958,7 @@ i915_gem_file_idle_work_handler(struct work_struct *work) int i915_gem_open(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv; + int ret; DRM_DEBUG_DRIVER("\n"); @@ -4959,15 +4968,18 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file) file->driver_priv = file_priv; file_priv->dev_priv = dev->dev_private; + file_priv->file = file; spin_lock_init(&file_priv->mm.lock); INIT_LIST_HEAD(&file_priv->mm.request_list); INIT_DELAYED_WORK(&file_priv->mm.idle_work, i915_gem_file_idle_work_handler); - idr_init(&file_priv->context_idr); + ret = i915_gem_context_open(dev, file); + if (ret) + kfree(file_priv); - return 0; + return ret; } static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) @@ -4983,27 +4995,46 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) #endif } +static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) +{ + if (!mutex_trylock(&dev->struct_mutex)) { + if (!mutex_is_locked_by(&dev->struct_mutex, current)) + return false; + + if (to_i915(dev)->mm.shrinker_no_lock_stealing) + return false; + + *unlock = false; + } else + *unlock = true; + + return true; +} + +static int num_vma_bound(struct drm_i915_gem_object *obj) +{ + struct i915_vma *vma; + int count = 0; + + list_for_each_entry(vma, &obj->vma_list, vma_link) + if (drm_mm_node_allocated(&vma->node)) + count++; + + return count; +} + static unsigned long -i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc) +i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { struct drm_i915_private *dev_priv = - container_of(shrinker, - struct drm_i915_private, - mm.inactive_shrinker); + container_of(shrinker, struct drm_i915_private, mm.shrinker); struct drm_device *dev = dev_priv->dev; struct drm_i915_gem_object *obj; - bool unlock = true; unsigned long count; + bool unlock; - if (!mutex_trylock(&dev->struct_mutex)) { - if (!mutex_is_locked_by(&dev->struct_mutex, current)) - return 0; - - if (dev_priv->mm.shrinker_no_lock_stealing) - return 0; - - unlock = false; - } + if (!i915_gem_shrinker_lock(dev, &unlock)) + return 0; count = 0; list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) @@ -5011,10 +5042,8 @@ i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc) count += obj->base.size >> PAGE_SHIFT; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { - if (obj->active) - continue; - - if (obj->pin_count == 0 && obj->pages_pin_count == 0) + if (!i915_gem_obj_is_pinned(obj) && + obj->pages_pin_count == num_vma_bound(obj)) count += obj->base.size >> PAGE_SHIFT; } @@ -5031,7 +5060,8 @@ unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o, struct drm_i915_private *dev_priv = o->base.dev->dev_private; struct i915_vma *vma; - if (vm == &dev_priv->mm.aliasing_ppgtt->base) + if (!dev_priv->mm.aliasing_ppgtt || + vm == &dev_priv->mm.aliasing_ppgtt->base) vm = &dev_priv->gtt.base; BUG_ON(list_empty(&o->vma_list)); @@ -5072,7 +5102,8 @@ unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, struct drm_i915_private *dev_priv = o->base.dev->dev_private; struct i915_vma *vma; - if (vm == &dev_priv->mm.aliasing_ppgtt->base) + if (!dev_priv->mm.aliasing_ppgtt || + vm == &dev_priv->mm.aliasing_ppgtt->base) vm = &dev_priv->gtt.base; BUG_ON(list_empty(&o->vma_list)); @@ -5085,49 +5116,104 @@ unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, } static unsigned long -i915_gem_inactive_scan(struct shrinker *shrinker, struct shrink_control *sc) +i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { struct drm_i915_private *dev_priv = - container_of(shrinker, - struct drm_i915_private, - mm.inactive_shrinker); + container_of(shrinker, struct drm_i915_private, mm.shrinker); struct drm_device *dev = dev_priv->dev; unsigned long freed; - bool unlock = true; - - if (!mutex_trylock(&dev->struct_mutex)) { - if (!mutex_is_locked_by(&dev->struct_mutex, current)) - return SHRINK_STOP; + bool unlock; - if (dev_priv->mm.shrinker_no_lock_stealing) - return SHRINK_STOP; - - unlock = false; - } + if (!i915_gem_shrinker_lock(dev, &unlock)) + return SHRINK_STOP; freed = i915_gem_purge(dev_priv, sc->nr_to_scan); if (freed < sc->nr_to_scan) freed += __i915_gem_shrink(dev_priv, sc->nr_to_scan - freed, false); - if (freed < sc->nr_to_scan) - freed += i915_gem_shrink_all(dev_priv); - if (unlock) mutex_unlock(&dev->struct_mutex); return freed; } +static int +i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct drm_i915_private *dev_priv = + container_of(nb, struct drm_i915_private, mm.oom_notifier); + struct drm_device *dev = dev_priv->dev; + struct drm_i915_gem_object *obj; + unsigned long timeout = msecs_to_jiffies(5000) + 1; + unsigned long pinned, bound, unbound, freed; + bool was_interruptible; + bool unlock; + + while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) + schedule_timeout_killable(1); + if (timeout == 0) { + pr_err("Unable to purge GPU memory due lock contention.\n"); + return NOTIFY_DONE; + } + + was_interruptible = dev_priv->mm.interruptible; + dev_priv->mm.interruptible = false; + + freed = i915_gem_shrink_all(dev_priv); + + dev_priv->mm.interruptible = was_interruptible; + + /* Because we may be allocating inside our own driver, we cannot + * assert that there are no objects with pinned pages that are not + * being pointed to by hardware. + */ + unbound = bound = pinned = 0; + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + if (!obj->base.filp) /* not backed by a freeable object */ + continue; + + if (obj->pages_pin_count) + pinned += obj->base.size; + else + unbound += obj->base.size; + } + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + if (!obj->base.filp) + continue; + + if (obj->pages_pin_count) + pinned += obj->base.size; + else + bound += obj->base.size; + } + + if (unlock) + mutex_unlock(&dev->struct_mutex); + + pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n", + freed, pinned); + if (unbound || bound) + pr_err("%lu and %lu bytes still available in the " + "bound and unbound GPU page lists.\n", + bound, unbound); + + *(unsigned long *)ptr += freed; + return NOTIFY_DONE; +} + struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj) { struct i915_vma *vma; + /* This WARN has probably outlived its usefulness (callers already + * WARN if they don't find the GGTT vma they expect). When removing, + * remember to remove the pre-check in is_pin_display() as well */ if (WARN_ON(list_empty(&obj->vma_list))) return NULL; vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link); - if (WARN_ON(vma->vm != obj_to_ggtt(obj))) + if (vma->vm != obj_to_ggtt(obj)) return NULL; return vma; diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index e08acaba540..a5ddf3bce9c 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -93,11 +93,60 @@ * I've seen in a spec to date, and that was a workaround for a non-shipping * part. It should be safe to decrease this, but it's more future proof as is. */ -#define CONTEXT_ALIGN (64<<10) +#define GEN6_CONTEXT_ALIGN (64<<10) +#define GEN7_CONTEXT_ALIGN 4096 -static struct i915_hw_context * -i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id); -static int do_switch(struct i915_hw_context *to); +static void do_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_address_space *vm = &ppgtt->base; + + if (ppgtt == dev_priv->mm.aliasing_ppgtt || + (list_empty(&vm->active_list) && list_empty(&vm->inactive_list))) { + ppgtt->base.cleanup(&ppgtt->base); + return; + } + + /* + * Make sure vmas are unbound before we take down the drm_mm + * + * FIXME: Proper refcounting should take care of this, this shouldn't be + * needed at all. + */ + if (!list_empty(&vm->active_list)) { + struct i915_vma *vma; + + list_for_each_entry(vma, &vm->active_list, mm_list) + if (WARN_ON(list_empty(&vma->vma_link) || + list_is_singular(&vma->vma_link))) + break; + + i915_gem_evict_vm(&ppgtt->base, true); + } else { + i915_gem_retire_requests(dev); + i915_gem_evict_vm(&ppgtt->base, false); + } + + ppgtt->base.cleanup(&ppgtt->base); +} + +static void ppgtt_release(struct kref *kref) +{ + struct i915_hw_ppgtt *ppgtt = + container_of(kref, struct i915_hw_ppgtt, ref); + + do_ppgtt_cleanup(ppgtt); + kfree(ppgtt); +} + +static size_t get_context_alignment(struct drm_device *dev) +{ + if (IS_GEN6(dev)) + return GEN6_CONTEXT_ALIGN; + + return GEN7_CONTEXT_ALIGN; +} static int get_context_size(struct drm_device *dev) { @@ -129,20 +178,52 @@ static int get_context_size(struct drm_device *dev) void i915_gem_context_free(struct kref *ctx_ref) { - struct i915_hw_context *ctx = container_of(ctx_ref, + struct intel_context *ctx = container_of(ctx_ref, typeof(*ctx), ref); + struct i915_hw_ppgtt *ppgtt = NULL; + + if (ctx->obj) { + /* We refcount even the aliasing PPGTT to keep the code symmetric */ + if (USES_PPGTT(ctx->obj->base.dev)) + ppgtt = ctx_to_ppgtt(ctx); + + /* XXX: Free up the object before tearing down the address space, in + * case we're bound in the PPGTT */ + drm_gem_object_unreference(&ctx->obj->base); + } + if (ppgtt) + kref_put(&ppgtt->ref, ppgtt_release); list_del(&ctx->link); - drm_gem_object_unreference(&ctx->obj->base); kfree(ctx); } -static struct i915_hw_context * -create_hw_context(struct drm_device *dev, +static struct i915_hw_ppgtt * +create_vm_for_ctx(struct drm_device *dev, struct intel_context *ctx) +{ + struct i915_hw_ppgtt *ppgtt; + int ret; + + ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); + if (!ppgtt) + return ERR_PTR(-ENOMEM); + + ret = i915_gem_init_ppgtt(dev, ppgtt); + if (ret) { + kfree(ppgtt); + return ERR_PTR(ret); + } + + ppgtt->ctx = ctx; + return ppgtt; +} + +static struct intel_context * +__create_hw_context(struct drm_device *dev, struct drm_i915_file_private *file_priv) { struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_hw_context *ctx; + struct intel_context *ctx; int ret; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); @@ -150,37 +231,40 @@ create_hw_context(struct drm_device *dev, return ERR_PTR(-ENOMEM); kref_init(&ctx->ref); - ctx->obj = i915_gem_alloc_object(dev, dev_priv->hw_context_size); - INIT_LIST_HEAD(&ctx->link); - if (ctx->obj == NULL) { - kfree(ctx); - DRM_DEBUG_DRIVER("Context object allocated failed\n"); - return ERR_PTR(-ENOMEM); - } + list_add_tail(&ctx->link, &dev_priv->context_list); - if (INTEL_INFO(dev)->gen >= 7) { - ret = i915_gem_object_set_cache_level(ctx->obj, - I915_CACHE_L3_LLC); - /* Failure shouldn't ever happen this early */ - if (WARN_ON(ret)) + if (dev_priv->hw_context_size) { + ctx->obj = i915_gem_alloc_object(dev, dev_priv->hw_context_size); + if (ctx->obj == NULL) { + ret = -ENOMEM; goto err_out; + } + + /* + * Try to make the context utilize L3 as well as LLC. + * + * On VLV we don't have L3 controls in the PTEs so we + * shouldn't touch the cache level, especially as that + * would make the object snooped which might have a + * negative performance impact. + */ + if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) { + ret = i915_gem_object_set_cache_level(ctx->obj, + I915_CACHE_L3_LLC); + /* Failure shouldn't ever happen this early */ + if (WARN_ON(ret)) + goto err_out; + } } - /* The ring associated with the context object is handled by the normal - * object tracking code. We give an initial ring value simple to pass an - * assertion in the context switch code. - */ - ctx->ring = &dev_priv->ring[RCS]; - list_add_tail(&ctx->link, &dev_priv->context_list); - /* Default context will never have a file_priv */ - if (file_priv == NULL) - return ctx; - - ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_ID + 1, 0, - GFP_KERNEL); - if (ret < 0) - goto err_out; + if (file_priv != NULL) { + ret = idr_alloc(&file_priv->context_idr, ctx, + DEFAULT_CONTEXT_ID, 0, GFP_KERNEL); + if (ret < 0) + goto err_out; + } else + ret = DEFAULT_CONTEXT_ID; ctx->file_priv = file_priv; ctx->id = ret; @@ -196,149 +280,244 @@ err_out: return ERR_PTR(ret); } -static inline bool is_default_context(struct i915_hw_context *ctx) -{ - return (ctx == ctx->ring->default_context); -} - /** * The default context needs to exist per ring that uses contexts. It stores the * context state of the GPU for applications that don't utilize HW contexts, as * well as an idle case. */ -static int create_default_context(struct drm_i915_private *dev_priv) +static struct intel_context * +i915_gem_create_context(struct drm_device *dev, + struct drm_i915_file_private *file_priv, + bool create_vm) { - struct i915_hw_context *ctx; - int ret; + const bool is_global_default_ctx = file_priv == NULL; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_context *ctx; + int ret = 0; - BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); + BUG_ON(!mutex_is_locked(&dev->struct_mutex)); - ctx = create_hw_context(dev_priv->dev, NULL); + ctx = __create_hw_context(dev, file_priv); if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - /* We may need to do things with the shrinker which require us to - * immediately switch back to the default context. This can cause a - * problem as pinning the default context also requires GTT space which - * may not be available. To avoid this we always pin the - * default context. - */ - ret = i915_gem_obj_ggtt_pin(ctx->obj, CONTEXT_ALIGN, false, false); - if (ret) { - DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret); - goto err_destroy; - } + return ctx; - ret = do_switch(ctx); - if (ret) { - DRM_DEBUG_DRIVER("Switch failed %d\n", ret); - goto err_unpin; + if (is_global_default_ctx && ctx->obj) { + /* We may need to do things with the shrinker which + * require us to immediately switch back to the default + * context. This can cause a problem as pinning the + * default context also requires GTT space which may not + * be available. To avoid this we always pin the default + * context. + */ + ret = i915_gem_obj_ggtt_pin(ctx->obj, + get_context_alignment(dev), 0); + if (ret) { + DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret); + goto err_destroy; + } } - dev_priv->ring[RCS].default_context = ctx; + if (create_vm) { + struct i915_hw_ppgtt *ppgtt = create_vm_for_ctx(dev, ctx); + + if (IS_ERR_OR_NULL(ppgtt)) { + DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", + PTR_ERR(ppgtt)); + ret = PTR_ERR(ppgtt); + goto err_unpin; + } else + ctx->vm = &ppgtt->base; + + /* This case is reserved for the global default context and + * should only happen once. */ + if (is_global_default_ctx) { + if (WARN_ON(dev_priv->mm.aliasing_ppgtt)) { + ret = -EEXIST; + goto err_unpin; + } + + dev_priv->mm.aliasing_ppgtt = ppgtt; + } + } else if (USES_PPGTT(dev)) { + /* For platforms which only have aliasing PPGTT, we fake the + * address space and refcounting. */ + ctx->vm = &dev_priv->mm.aliasing_ppgtt->base; + kref_get(&dev_priv->mm.aliasing_ppgtt->ref); + } else + ctx->vm = &dev_priv->gtt.base; - DRM_DEBUG_DRIVER("Default HW context loaded\n"); - return 0; + return ctx; err_unpin: - i915_gem_object_unpin(ctx->obj); + if (is_global_default_ctx && ctx->obj) + i915_gem_object_ggtt_unpin(ctx->obj); err_destroy: i915_gem_context_unreference(ctx); - return ret; + return ERR_PTR(ret); } -int i915_gem_context_init(struct drm_device *dev) +void i915_gem_context_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - int ret; + int i; - if (!HAS_HW_CONTEXTS(dev)) - return 0; + /* Prevent the hardware from restoring the last context (which hung) on + * the next switch */ + for (i = 0; i < I915_NUM_RINGS; i++) { + struct intel_engine_cs *ring = &dev_priv->ring[i]; + struct intel_context *dctx = ring->default_context; - /* If called from reset, or thaw... we've been here already */ - if (dev_priv->ring[RCS].default_context) - return 0; + /* Do a fake switch to the default context */ + if (ring->last_context == dctx) + continue; - dev_priv->hw_context_size = round_up(get_context_size(dev), 4096); + if (!ring->last_context) + continue; - if (dev_priv->hw_context_size > (1<<20)) { - DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size\n"); - return -E2BIG; + if (dctx->obj && i == RCS) { + WARN_ON(i915_gem_obj_ggtt_pin(dctx->obj, + get_context_alignment(dev), 0)); + /* Fake a finish/inactive */ + dctx->obj->base.write_domain = 0; + dctx->obj->active = 0; + } + + i915_gem_context_unreference(ring->last_context); + i915_gem_context_reference(dctx); + ring->last_context = dctx; } +} - ret = create_default_context(dev_priv); - if (ret) { - DRM_DEBUG_DRIVER("Disabling HW Contexts; create failed %d\n", - ret); - return ret; +int i915_gem_context_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_context *ctx; + int i; + + /* Init should only be called once per module load. Eventually the + * restriction on the context_disabled check can be loosened. */ + if (WARN_ON(dev_priv->ring[RCS].default_context)) + return 0; + + if (HAS_HW_CONTEXTS(dev)) { + dev_priv->hw_context_size = round_up(get_context_size(dev), 4096); + if (dev_priv->hw_context_size > (1<<20)) { + DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size %d\n", + dev_priv->hw_context_size); + dev_priv->hw_context_size = 0; + } } - DRM_DEBUG_DRIVER("HW context support initialized\n"); + ctx = i915_gem_create_context(dev, NULL, USES_PPGTT(dev)); + if (IS_ERR(ctx)) { + DRM_ERROR("Failed to create default global context (error %ld)\n", + PTR_ERR(ctx)); + return PTR_ERR(ctx); + } + + /* NB: RCS will hold a ref for all rings */ + for (i = 0; i < I915_NUM_RINGS; i++) + dev_priv->ring[i].default_context = ctx; + + DRM_DEBUG_DRIVER("%s context support initialized\n", dev_priv->hw_context_size ? "HW" : "fake"); return 0; } void i915_gem_context_fini(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_hw_context *dctx = dev_priv->ring[RCS].default_context; + struct intel_context *dctx = dev_priv->ring[RCS].default_context; + int i; + + if (dctx->obj) { + /* The only known way to stop the gpu from accessing the hw context is + * to reset it. Do this as the very last operation to avoid confusing + * other code, leading to spurious errors. */ + intel_gpu_reset(dev); + + /* When default context is created and switched to, base object refcount + * will be 2 (+1 from object creation and +1 from do_switch()). + * i915_gem_context_fini() will be called after gpu_idle() has switched + * to default context. So we need to unreference the base object once + * to offset the do_switch part, so that i915_gem_context_unreference() + * can then free the base object correctly. */ + WARN_ON(!dev_priv->ring[RCS].last_context); + if (dev_priv->ring[RCS].last_context == dctx) { + /* Fake switch to NULL context */ + WARN_ON(dctx->obj->active); + i915_gem_object_ggtt_unpin(dctx->obj); + i915_gem_context_unreference(dctx); + dev_priv->ring[RCS].last_context = NULL; + } + + i915_gem_object_ggtt_unpin(dctx->obj); + } - if (!HAS_HW_CONTEXTS(dev)) - return; + for (i = 0; i < I915_NUM_RINGS; i++) { + struct intel_engine_cs *ring = &dev_priv->ring[i]; - /* The only known way to stop the gpu from accessing the hw context is - * to reset it. Do this as the very last operation to avoid confusing - * other code, leading to spurious errors. */ - intel_gpu_reset(dev); - - /* When default context is created and switched to, base object refcount - * will be 2 (+1 from object creation and +1 from do_switch()). - * i915_gem_context_fini() will be called after gpu_idle() has switched - * to default context. So we need to unreference the base object once - * to offset the do_switch part, so that i915_gem_context_unreference() - * can then free the base object correctly. */ - WARN_ON(!dev_priv->ring[RCS].last_context); - if (dev_priv->ring[RCS].last_context == dctx) { - /* Fake switch to NULL context */ - WARN_ON(dctx->obj->active); - i915_gem_object_unpin(dctx->obj); - i915_gem_context_unreference(dctx); + if (ring->last_context) + i915_gem_context_unreference(ring->last_context); + + ring->default_context = NULL; + ring->last_context = NULL; } - i915_gem_object_unpin(dctx->obj); i915_gem_context_unreference(dctx); - dev_priv->ring[RCS].default_context = NULL; - dev_priv->ring[RCS].last_context = NULL; } -static int context_idr_cleanup(int id, void *p, void *data) +int i915_gem_context_enable(struct drm_i915_private *dev_priv) { - struct i915_hw_context *ctx = p; + struct intel_engine_cs *ring; + int ret, i; + + /* This is the only place the aliasing PPGTT gets enabled, which means + * it has to happen before we bail on reset */ + if (dev_priv->mm.aliasing_ppgtt) { + struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; + ppgtt->enable(ppgtt); + } - BUG_ON(id == DEFAULT_CONTEXT_ID); + /* FIXME: We should make this work, even in reset */ + if (i915_reset_in_progress(&dev_priv->gpu_error)) + return 0; + + BUG_ON(!dev_priv->ring[RCS].default_context); + + for_each_ring(ring, dev_priv, i) { + ret = i915_switch_context(ring, ring->default_context); + if (ret) + return ret; + } + + return 0; +} + +static int context_idr_cleanup(int id, void *p, void *data) +{ + struct intel_context *ctx = p; i915_gem_context_unreference(ctx); return 0; } -struct i915_ctx_hang_stats * -i915_gem_context_get_hang_stats(struct drm_device *dev, - struct drm_file *file, - u32 id) +int i915_gem_context_open(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; - struct i915_hw_context *ctx; + struct intel_context *ctx; - if (id == DEFAULT_CONTEXT_ID) - return &file_priv->hang_stats; + idr_init(&file_priv->context_idr); - if (!HAS_HW_CONTEXTS(dev)) - return ERR_PTR(-ENOENT); + mutex_lock(&dev->struct_mutex); + ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev)); + mutex_unlock(&dev->struct_mutex); - ctx = i915_gem_context_get(file->driver_priv, id); - if (ctx == NULL) - return ERR_PTR(-ENOENT); + if (IS_ERR(ctx)) { + idr_destroy(&file_priv->context_idr); + return PTR_ERR(ctx); + } - return &ctx->hang_stats; + return 0; } void i915_gem_context_close(struct drm_device *dev, struct drm_file *file) @@ -349,15 +528,21 @@ void i915_gem_context_close(struct drm_device *dev, struct drm_file *file) idr_destroy(&file_priv->context_idr); } -static struct i915_hw_context * +struct intel_context * i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id) { - return (struct i915_hw_context *)idr_find(&file_priv->context_idr, id); + struct intel_context *ctx; + + ctx = (struct intel_context *)idr_find(&file_priv->context_idr, id); + if (!ctx) + return ERR_PTR(-ENOENT); + + return ctx; } static inline int -mi_set_context(struct intel_ring_buffer *ring, - struct i915_hw_context *new_context, +mi_set_context(struct intel_engine_cs *ring, + struct intel_context *new_context, u32 hw_flags) { int ret; @@ -367,7 +552,7 @@ mi_set_context(struct intel_ring_buffer *ring, * explicitly, so we rely on the value at ring init, stored in * itlb_before_ctx_switch. */ - if (IS_GEN6(ring->dev) && ring->itlb_before_ctx_switch) { + if (IS_GEN6(ring->dev)) { ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, 0); if (ret) return ret; @@ -377,8 +562,8 @@ mi_set_context(struct intel_ring_buffer *ring, if (ret) return ret; - /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw */ - if (IS_GEN7(ring->dev)) + /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ + if (INTEL_INFO(ring->dev)->gen >= 7) intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE); else intel_ring_emit(ring, MI_NOOP); @@ -390,10 +575,13 @@ mi_set_context(struct intel_ring_buffer *ring, MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN | hw_flags); - /* w/a: MI_SET_CONTEXT must always be followed by MI_NOOP */ + /* + * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP + * WaMiSetContext_Hang:snb,ivb,vlv + */ intel_ring_emit(ring, MI_NOOP); - if (IS_GEN7(ring->dev)) + if (INTEL_INFO(ring->dev)->gen >= 7) intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_ENABLE); else intel_ring_emit(ring, MI_NOOP); @@ -403,21 +591,31 @@ mi_set_context(struct intel_ring_buffer *ring, return ret; } -static int do_switch(struct i915_hw_context *to) +static int do_switch(struct intel_engine_cs *ring, + struct intel_context *to) { - struct intel_ring_buffer *ring = to->ring; - struct i915_hw_context *from = ring->last_context; + struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct intel_context *from = ring->last_context; + struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(to); u32 hw_flags = 0; + bool uninitialized = false; int ret, i; - BUG_ON(from != NULL && from->obj != NULL && from->obj->pin_count == 0); + if (from != NULL && ring == &dev_priv->ring[RCS]) { + BUG_ON(from->obj == NULL); + BUG_ON(!i915_gem_obj_is_pinned(from->obj)); + } - if (from == to && !to->remap_slice) + if (from == to && from->last_ring == ring && !to->remap_slice) return 0; - ret = i915_gem_obj_ggtt_pin(to->obj, CONTEXT_ALIGN, false, false); - if (ret) - return ret; + /* Trying to pin first makes error handling easier. */ + if (ring == &dev_priv->ring[RCS]) { + ret = i915_gem_obj_ggtt_pin(to->obj, + get_context_alignment(ring->dev), 0); + if (ret) + return ret; + } /* * Pin can switch back to the default context if we end up calling into @@ -426,6 +624,18 @@ static int do_switch(struct i915_hw_context *to) */ from = ring->last_context; + if (USES_FULL_PPGTT(ring->dev)) { + ret = ppgtt->switch_mm(ppgtt, ring, false); + if (ret) + goto unpin_out; + } + + if (ring != &dev_priv->ring[RCS]) { + if (from) + i915_gem_context_unreference(from); + goto done; + } + /* * Clear this page out of any CPU caches for coherent swap-in/out. Note * that thanks to write = false in this call and us not setting any gpu @@ -435,22 +645,21 @@ static int do_switch(struct i915_hw_context *to) * XXX: We need a real interface to do this instead of trickery. */ ret = i915_gem_object_set_to_gtt_domain(to->obj, false); - if (ret) { - i915_gem_object_unpin(to->obj); - return ret; - } + if (ret) + goto unpin_out; - if (!to->obj->has_global_gtt_mapping) - i915_gem_gtt_bind_object(to->obj, to->obj->cache_level); + if (!to->obj->has_global_gtt_mapping) { + struct i915_vma *vma = i915_gem_obj_to_vma(to->obj, + &dev_priv->gtt.base); + vma->bind_vma(vma, to->obj->cache_level, GLOBAL_BIND); + } - if (!to->is_initialized || is_default_context(to)) + if (!to->is_initialized || i915_gem_context_is_default(to)) hw_flags |= MI_RESTORE_INHIBIT; ret = mi_set_context(ring, to, hw_flags); - if (ret) { - i915_gem_object_unpin(to->obj); - return ret; - } + if (ret) + goto unpin_out; for (i = 0; i < MAX_L3_SLICES; i++) { if (!(to->remap_slice & (1<<i))) @@ -484,55 +693,65 @@ static int do_switch(struct i915_hw_context *to) BUG_ON(from->obj->ring != ring); /* obj is kept alive until the next request by its active ref */ - i915_gem_object_unpin(from->obj); + i915_gem_object_ggtt_unpin(from->obj); i915_gem_context_unreference(from); } + uninitialized = !to->is_initialized && from == NULL; + to->is_initialized = true; + +done: i915_gem_context_reference(to); ring->last_context = to; - to->is_initialized = true; + to->last_ring = ring; + + if (uninitialized) { + ret = i915_gem_render_state_init(ring); + if (ret) + DRM_ERROR("init render state: %d\n", ret); + } return 0; + +unpin_out: + if (ring->id == RCS) + i915_gem_object_ggtt_unpin(to->obj); + return ret; } /** * i915_switch_context() - perform a GPU context switch. * @ring: ring for which we'll execute the context switch - * @file_priv: file_priv associated with the context, may be NULL - * @id: context id number + * @to: the context to switch to * * The context life cycle is simple. The context refcount is incremented and * decremented by 1 and create and destroy. If the context is in use by the GPU, * it will have a refoucnt > 1. This allows us to destroy the context abstract * object while letting the normal object tracking destroy the backing BO. */ -int i915_switch_context(struct intel_ring_buffer *ring, - struct drm_file *file, - int to_id) +int i915_switch_context(struct intel_engine_cs *ring, + struct intel_context *to) { struct drm_i915_private *dev_priv = ring->dev->dev_private; - struct i915_hw_context *to; - - if (!HAS_HW_CONTEXTS(ring->dev)) - return 0; WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); - if (ring != &dev_priv->ring[RCS]) + if (to->obj == NULL) { /* We have the fake context */ + if (to != ring->last_context) { + i915_gem_context_reference(to); + if (ring->last_context) + i915_gem_context_unreference(ring->last_context); + ring->last_context = to; + } return 0; - - if (to_id == DEFAULT_CONTEXT_ID) { - to = ring->default_context; - } else { - if (file == NULL) - return -EINVAL; - - to = i915_gem_context_get(file->driver_priv, to_id); - if (to == NULL) - return -ENOENT; } - return do_switch(to); + return do_switch(ring, to); +} + +static bool hw_context_enabled(struct drm_device *dev) +{ + return to_i915(dev)->hw_context_size; } int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, @@ -540,20 +759,17 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_context_create *args = data; struct drm_i915_file_private *file_priv = file->driver_priv; - struct i915_hw_context *ctx; + struct intel_context *ctx; int ret; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; - - if (!HAS_HW_CONTEXTS(dev)) + if (!hw_context_enabled(dev)) return -ENODEV; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - ctx = create_hw_context(dev, file_priv); + ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev)); mutex_unlock(&dev->struct_mutex); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -569,20 +785,20 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_context_destroy *args = data; struct drm_i915_file_private *file_priv = file->driver_priv; - struct i915_hw_context *ctx; + struct intel_context *ctx; int ret; - if (!(dev->driver->driver_features & DRIVER_GEM)) - return -ENODEV; + if (args->ctx_id == DEFAULT_CONTEXT_ID) + return -ENOENT; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; ctx = i915_gem_context_get(file_priv, args->ctx_id); - if (!ctx) { + if (IS_ERR(ctx)) { mutex_unlock(&dev->struct_mutex); - return -ENOENT; + return PTR_ERR(ctx); } idr_remove(&ctx->file_priv->context_idr, ctx->id); diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c index 775d506b320..f462d1b51d9 100644 --- a/drivers/gpu/drm/i915/i915_gem_debug.c +++ b/drivers/gpu/drm/i915/i915_gem_debug.c @@ -34,7 +34,7 @@ int i915_verify_lists(struct drm_device *dev) { static int warned; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; int err = 0; diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c index 9bb533e0d76..580aa42443e 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c @@ -161,12 +161,8 @@ static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; - int ret; - - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return; + mutex_lock(&dev->struct_mutex); if (--obj->vmapping_count == 0) { vunmap(obj->dma_buf_vmapping); obj->dma_buf_vmapping = NULL; @@ -233,6 +229,14 @@ static const struct dma_buf_ops i915_dmabuf_ops = { struct dma_buf *i915_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gem_obj, int flags) { + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); + + if (obj->ops->dmabuf_export) { + int ret = obj->ops->dmabuf_export(obj); + if (ret) + return ERR_PTR(ret); + } + return dma_buf_export(gem_obj, &i915_dmabuf_ops, gem_obj->size, flags); } diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index 2ca280f9ee5..bbf4b12d842 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -36,7 +36,7 @@ static bool mark_free(struct i915_vma *vma, struct list_head *unwind) { - if (vma->obj->pin_count) + if (vma->pin_count) return false; if (WARN_ON(!list_empty(&vma->exec_list))) @@ -46,18 +46,37 @@ mark_free(struct i915_vma *vma, struct list_head *unwind) return drm_mm_scan_add_block(&vma->node); } +/** + * i915_gem_evict_something - Evict vmas to make room for binding a new one + * @dev: drm_device + * @vm: address space to evict from + * @size: size of the desired free space + * @alignment: alignment constraint of the desired free space + * @cache_level: cache_level for the desired space + * @mappable: whether the free space must be mappable + * @nonblocking: whether evicting active objects is allowed or not + * + * This function will try to evict vmas until a free space satisfying the + * requirements is found. Callers must check first whether any such hole exists + * already before calling this function. + * + * This function is used by the object/vma binding code. + * + * To clarify: This is for freeing up virtual address space, not for freeing + * memory in e.g. the shrinker. + */ int i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm, int min_size, unsigned alignment, unsigned cache_level, - bool mappable, bool nonblocking) + unsigned long start, unsigned long end, + unsigned flags) { - drm_i915_private_t *dev_priv = dev->dev_private; struct list_head eviction_list, unwind_list; struct i915_vma *vma; int ret = 0; int pass = 0; - trace_i915_gem_evict(dev, min_size, alignment, mappable); + trace_i915_gem_evict(dev, min_size, alignment, flags); /* * The goal is to evict objects and amalgamate space in LRU order. @@ -83,11 +102,10 @@ i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm, */ INIT_LIST_HEAD(&unwind_list); - if (mappable) { - BUG_ON(!i915_is_ggtt(vm)); + if (start != 0 || end != vm->total) { drm_mm_init_scan_with_range(&vm->mm, min_size, - alignment, cache_level, 0, - dev_priv->gtt.mappable_end); + alignment, cache_level, + start, end); } else drm_mm_init_scan(&vm->mm, min_size, alignment, cache_level); @@ -98,7 +116,7 @@ search_again: goto found; } - if (nonblocking) + if (flags & PIN_NONBLOCK) goto none; /* Now merge in the soon-to-be-expired objects... */ @@ -122,7 +140,7 @@ none: /* Can we unpin some objects such as idle hw contents, * or pending flips? */ - if (nonblocking) + if (flags & PIN_NONBLOCK) return -ENOSPC; /* Only idle the GPU and repeat the search once */ @@ -177,19 +195,19 @@ found: } /** - * i915_gem_evict_vm - Try to free up VM space + * i915_gem_evict_vm - Evict all idle vmas from a vm * - * @vm: Address space to evict from + * @vm: Address space to cleanse * @do_idle: Boolean directing whether to idle first. * - * VM eviction is about freeing up virtual address space. If one wants fine - * grained eviction, they should see evict something for more details. In terms - * of freeing up actual system memory, this function may not accomplish the - * desired result. An object may be shared in multiple address space, and this - * function will not assert those objects be freed. + * This function evicts all idles vmas from a vm. If all unpinned vmas should be + * evicted the @do_idle needs to be set to true. * - * Using do_idle will result in a more complete eviction because it retires, and - * inactivates current BOs. + * This is used by the execbuf code as a last-ditch effort to defragment the + * address space. + * + * To clarify: This is for freeing up virtual address space, not for freeing + * memory in e.g. the shrinker. */ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle) { @@ -207,16 +225,24 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle) } list_for_each_entry_safe(vma, next, &vm->inactive_list, mm_list) - if (vma->obj->pin_count == 0) + if (vma->pin_count == 0) WARN_ON(i915_vma_unbind(vma)); return 0; } +/** + * i915_gem_evict_everything - Try to evict all objects + * @dev: Device to evict objects for + * + * This functions tries to evict all gem objects from all address spaces. Used + * by the shrinker as a last-ditch effort and for suspend, before releasing the + * backing storage of all unbound objects. + */ int i915_gem_evict_everything(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct i915_address_space *vm; bool lists_empty = true; int ret; diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index d269ecf46e2..3a30133f93e 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -35,6 +35,9 @@ #define __EXEC_OBJECT_HAS_PIN (1<<31) #define __EXEC_OBJECT_HAS_FENCE (1<<30) +#define __EXEC_OBJECT_NEEDS_BIAS (1<<28) + +#define BATCH_OFFSET_BIAS (256*1024) struct eb_vmas { struct list_head vmas; @@ -91,6 +94,7 @@ eb_lookup_vmas(struct eb_vmas *eb, struct i915_address_space *vm, struct drm_file *file) { + struct drm_i915_private *dev_priv = vm->dev->dev_private; struct drm_i915_gem_object *obj; struct list_head objects; int i, ret; @@ -125,6 +129,20 @@ eb_lookup_vmas(struct eb_vmas *eb, i = 0; while (!list_empty(&objects)) { struct i915_vma *vma; + struct i915_address_space *bind_vm = vm; + + if (exec[i].flags & EXEC_OBJECT_NEEDS_GTT && + USES_FULL_PPGTT(vm->dev)) { + ret = -EINVAL; + goto err; + } + + /* If we have secure dispatch, or the userspace assures us that + * they know what they're doing, use the GGTT VM. + */ + if (((args->flags & I915_EXEC_SECURE) && + (i == (args->buffer_count - 1)))) + bind_vm = &dev_priv->gtt.base; obj = list_first_entry(&objects, struct drm_i915_gem_object, @@ -138,7 +156,7 @@ eb_lookup_vmas(struct eb_vmas *eb, * from the (obj, vm) we don't run the risk of creating * duplicated vmas for the same vm. */ - vma = i915_gem_obj_lookup_or_create_vma(obj, vm); + vma = i915_gem_obj_lookup_or_create_vma(obj, bind_vm); if (IS_ERR(vma)) { DRM_DEBUG("Failed to lookup VMA\n"); ret = PTR_ERR(vma); @@ -217,7 +235,7 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma) i915_gem_object_unpin_fence(obj); if (entry->flags & __EXEC_OBJECT_HAS_PIN) - i915_gem_object_unpin(obj); + vma->pin_count--; entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN); } @@ -247,10 +265,12 @@ static inline int use_cpu_reloc(struct drm_i915_gem_object *obj) static int relocate_entry_cpu(struct drm_i915_gem_object *obj, - struct drm_i915_gem_relocation_entry *reloc) + struct drm_i915_gem_relocation_entry *reloc, + uint64_t target_offset) { struct drm_device *dev = obj->base.dev; uint32_t page_offset = offset_in_page(reloc->offset); + uint64_t delta = reloc->delta + target_offset; char *vaddr; int ret; @@ -260,7 +280,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj, vaddr = kmap_atomic(i915_gem_object_get_page(obj, reloc->offset >> PAGE_SHIFT)); - *(uint32_t *)(vaddr + page_offset) = reloc->delta; + *(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta); if (INTEL_INFO(dev)->gen >= 8) { page_offset = offset_in_page(page_offset + sizeof(uint32_t)); @@ -271,7 +291,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj, (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT)); } - *(uint32_t *)(vaddr + page_offset) = 0; + *(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta); } kunmap_atomic(vaddr); @@ -281,10 +301,12 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj, static int relocate_entry_gtt(struct drm_i915_gem_object *obj, - struct drm_i915_gem_relocation_entry *reloc) + struct drm_i915_gem_relocation_entry *reloc, + uint64_t target_offset) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; + uint64_t delta = reloc->delta + target_offset; uint32_t __iomem *reloc_entry; void __iomem *reloc_page; int ret; @@ -303,7 +325,7 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj, reloc->offset & PAGE_MASK); reloc_entry = (uint32_t __iomem *) (reloc_page + offset_in_page(reloc->offset)); - iowrite32(reloc->delta, reloc_entry); + iowrite32(lower_32_bits(delta), reloc_entry); if (INTEL_INFO(dev)->gen >= 8) { reloc_entry += 1; @@ -316,7 +338,7 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj, reloc_entry = reloc_page; } - iowrite32(0, reloc_entry); + iowrite32(upper_32_bits(delta), reloc_entry); } io_mapping_unmap_atomic(reloc_page); @@ -327,14 +349,13 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj, static int i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, struct eb_vmas *eb, - struct drm_i915_gem_relocation_entry *reloc, - struct i915_address_space *vm) + struct drm_i915_gem_relocation_entry *reloc) { struct drm_device *dev = obj->base.dev; struct drm_gem_object *target_obj; struct drm_i915_gem_object *target_i915_obj; struct i915_vma *target_vma; - uint32_t target_offset; + uint64_t target_offset; int ret; /* we've already hold a reference to all valid objects */ @@ -352,8 +373,10 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, if (unlikely(IS_GEN6(dev) && reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && !target_i915_obj->has_global_gtt_mapping)) { - i915_gem_gtt_bind_object(target_i915_obj, - target_i915_obj->cache_level); + struct i915_vma *vma = + list_first_entry(&target_i915_obj->vma_list, + typeof(*vma), vma_link); + vma->bind_vma(vma, target_i915_obj->cache_level, GLOBAL_BIND); } /* Validate that the target is in a valid r/w GPU domain */ @@ -410,11 +433,10 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, if (obj->active && in_atomic()) return -EFAULT; - reloc->delta += target_offset; if (use_cpu_reloc(obj)) - ret = relocate_entry_cpu(obj, reloc); + ret = relocate_entry_cpu(obj, reloc, target_offset); else - ret = relocate_entry_gtt(obj, reloc); + ret = relocate_entry_gtt(obj, reloc, target_offset); if (ret) return ret; @@ -451,8 +473,7 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma, do { u64 offset = r->presumed_offset; - ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r, - vma->vm); + ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r); if (ret) return ret; @@ -481,8 +502,7 @@ i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma, int i, ret; for (i = 0; i < entry->relocation_count; i++) { - ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i], - vma->vm); + ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]); if (ret) return ret; } @@ -524,24 +544,31 @@ need_reloc_mappable(struct i915_vma *vma) static int i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, - struct intel_ring_buffer *ring, + struct intel_engine_cs *ring, bool *need_reloc) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct drm_i915_gem_object *obj = vma->obj; struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; - bool need_fence, need_mappable; - struct drm_i915_gem_object *obj = vma->obj; + bool need_fence; + uint64_t flags; int ret; + flags = 0; + need_fence = has_fenced_gpu_access && entry->flags & EXEC_OBJECT_NEEDS_FENCE && obj->tiling_mode != I915_TILING_NONE; - need_mappable = need_fence || need_reloc_mappable(vma); + if (need_fence || need_reloc_mappable(vma)) + flags |= PIN_MAPPABLE; + + if (entry->flags & EXEC_OBJECT_NEEDS_GTT) + flags |= PIN_GLOBAL; + if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) + flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; - ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, need_mappable, - false); + ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags); if (ret) return ret; @@ -560,14 +587,6 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, } } - /* Ensure ppgtt mapping exists if needed */ - if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) { - i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, - obj, obj->cache_level); - - obj->has_aliasing_ppgtt_mapping = 1; - } - if (entry->offset != vma->node.start) { entry->offset = vma->node.start; *need_reloc = true; @@ -578,15 +597,41 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma, obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER; } - if (entry->flags & EXEC_OBJECT_NEEDS_GTT && - !obj->has_global_gtt_mapping) - i915_gem_gtt_bind_object(obj, obj->cache_level); - return 0; } +static bool +eb_vma_misplaced(struct i915_vma *vma, bool has_fenced_gpu_access) +{ + struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; + struct drm_i915_gem_object *obj = vma->obj; + bool need_fence, need_mappable; + + need_fence = + has_fenced_gpu_access && + entry->flags & EXEC_OBJECT_NEEDS_FENCE && + obj->tiling_mode != I915_TILING_NONE; + need_mappable = need_fence || need_reloc_mappable(vma); + + WARN_ON((need_mappable || need_fence) && + !i915_is_ggtt(vma->vm)); + + if (entry->alignment && + vma->node.start & (entry->alignment - 1)) + return true; + + if (need_mappable && !obj->map_and_fenceable) + return true; + + if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS && + vma->node.start < BATCH_OFFSET_BIAS) + return true; + + return false; +} + static int -i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, +i915_gem_execbuffer_reserve(struct intel_engine_cs *ring, struct list_head *vmas, bool *need_relocs) { @@ -600,6 +645,8 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, if (list_empty(vmas)) return 0; + i915_gem_retire_requests_ring(ring); + vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm; INIT_LIST_HEAD(&ordered_vmas); @@ -646,26 +693,10 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, /* Unbind any ill-fitting objects or pin. */ list_for_each_entry(vma, vmas, exec_list) { - struct drm_i915_gem_exec_object2 *entry = vma->exec_entry; - bool need_fence, need_mappable; - - obj = vma->obj; - if (!drm_mm_node_allocated(&vma->node)) continue; - need_fence = - has_fenced_gpu_access && - entry->flags & EXEC_OBJECT_NEEDS_FENCE && - obj->tiling_mode != I915_TILING_NONE; - need_mappable = need_fence || need_reloc_mappable(vma); - - WARN_ON((need_mappable || need_fence) && - !i915_is_ggtt(vma->vm)); - - if ((entry->alignment && - vma->node.start & (entry->alignment - 1)) || - (need_mappable && !obj->map_and_fenceable)) + if (eb_vma_misplaced(vma, has_fenced_gpu_access)) ret = i915_vma_unbind(vma); else ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs); @@ -701,7 +732,7 @@ static int i915_gem_execbuffer_relocate_slow(struct drm_device *dev, struct drm_i915_gem_execbuffer2 *args, struct drm_file *file, - struct intel_ring_buffer *ring, + struct intel_engine_cs *ring, struct eb_vmas *eb, struct drm_i915_gem_exec_object2 *exec) { @@ -766,9 +797,9 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, * relocations were valid. */ for (j = 0; j < exec[i].relocation_count; j++) { - if (copy_to_user(&user_relocs[j].presumed_offset, - &invalid_offset, - sizeof(invalid_offset))) { + if (__copy_to_user(&user_relocs[j].presumed_offset, + &invalid_offset, + sizeof(invalid_offset))) { ret = -EFAULT; mutex_lock(&dev->struct_mutex); goto err; @@ -817,7 +848,7 @@ err: } static int -i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, +i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring, struct list_head *vmas) { struct i915_vma *vma; @@ -891,7 +922,7 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec, if (!access_ok(VERIFY_WRITE, ptr, length)) return -EFAULT; - if (likely(!i915_prefault_disable)) { + if (likely(!i915.prefault_disable)) { if (fault_in_multipages_readable(ptr, length)) return -EFAULT; } @@ -900,27 +931,32 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec, return 0; } -static int +static struct intel_context * i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, - const u32 ctx_id) + struct intel_engine_cs *ring, const u32 ctx_id) { + struct intel_context *ctx = NULL; struct i915_ctx_hang_stats *hs; - hs = i915_gem_context_get_hang_stats(dev, file, ctx_id); - if (IS_ERR(hs)) - return PTR_ERR(hs); + if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_ID) + return ERR_PTR(-EINVAL); + + ctx = i915_gem_context_get(file->driver_priv, ctx_id); + if (IS_ERR(ctx)) + return ctx; + hs = &ctx->hang_stats; if (hs->banned) { DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id); - return -EIO; + return ERR_PTR(-EIO); } - return 0; + return ctx; } static void i915_gem_execbuffer_move_to_active(struct list_head *vmas, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { struct i915_vma *vma; @@ -939,8 +975,13 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas, if (obj->base.write_domain) { obj->dirty = 1; obj->last_write_seqno = intel_ring_get_seqno(ring); - if (obj->pin_count) /* check for potential scanout */ + /* check for potential scanout */ + if (i915_gem_obj_ggtt_bound(obj) && + i915_gem_obj_to_ggtt(obj)->pin_count) intel_mark_fb_busy(obj, ring); + + /* update for the implicit flush after a batch */ + obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; } trace_i915_gem_object_change_domain(obj, old_read, old_write); @@ -950,7 +991,7 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas, static void i915_gem_execbuffer_retire_commands(struct drm_device *dev, struct drm_file *file, - struct intel_ring_buffer *ring, + struct intel_engine_cs *ring, struct drm_i915_gem_object *obj) { /* Unconditionally force add_request to emit a full flush. */ @@ -962,13 +1003,15 @@ i915_gem_execbuffer_retire_commands(struct drm_device *dev, static int i915_reset_gen7_sol_offsets(struct drm_device *dev, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int ret, i; - if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) - return 0; + if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) { + DRM_DEBUG("sol reset is gen7/rcs only\n"); + return -EINVAL; + } ret = intel_ring_begin(ring, 4 * 3); if (ret) @@ -985,20 +1028,71 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev, return 0; } +/** + * Find one BSD ring to dispatch the corresponding BSD command. + * The Ring ID is returned. + */ +static int gen8_dispatch_bsd_ring(struct drm_device *dev, + struct drm_file *file) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_file_private *file_priv = file->driver_priv; + + /* Check whether the file_priv is using one ring */ + if (file_priv->bsd_ring) + return file_priv->bsd_ring->id; + else { + /* If no, use the ping-pong mechanism to select one ring */ + int ring_id; + + mutex_lock(&dev->struct_mutex); + if (dev_priv->mm.bsd_ring_dispatch_index == 0) { + ring_id = VCS; + dev_priv->mm.bsd_ring_dispatch_index = 1; + } else { + ring_id = VCS2; + dev_priv->mm.bsd_ring_dispatch_index = 0; + } + file_priv->bsd_ring = &dev_priv->ring[ring_id]; + mutex_unlock(&dev->struct_mutex); + return ring_id; + } +} + +static struct drm_i915_gem_object * +eb_get_batch(struct eb_vmas *eb) +{ + struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list); + + /* + * SNA is doing fancy tricks with compressing batch buffers, which leads + * to negative relocation deltas. Usually that works out ok since the + * relocate address is still positive, except when the batch is placed + * very low in the GTT. Ensure this doesn't happen. + * + * Note that actual hangs have only been observed on gen7, but for + * paranoia do it everywhere. + */ + vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS; + + return vma->obj; +} + static int i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct drm_file *file, struct drm_i915_gem_execbuffer2 *args, - struct drm_i915_gem_exec_object2 *exec, - struct i915_address_space *vm) + struct drm_i915_gem_exec_object2 *exec) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct eb_vmas *eb; struct drm_i915_gem_object *batch_obj; struct drm_clip_rect *cliprects = NULL; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; + struct intel_context *ctx; + struct i915_address_space *vm; const u32 ctx_id = i915_execbuffer2_get_context_id(*args); - u32 exec_start, exec_len; + u64 exec_start = args->batch_start_offset, exec_len; u32 mask, flags; int ret, mode, i; bool need_relocs; @@ -1020,41 +1114,24 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, if (args->flags & I915_EXEC_IS_PINNED) flags |= I915_DISPATCH_PINNED; - switch (args->flags & I915_EXEC_RING_MASK) { - case I915_EXEC_DEFAULT: - case I915_EXEC_RENDER: - ring = &dev_priv->ring[RCS]; - break; - case I915_EXEC_BSD: - ring = &dev_priv->ring[VCS]; - if (ctx_id != DEFAULT_CONTEXT_ID) { - DRM_DEBUG("Ring %s doesn't support contexts\n", - ring->name); - return -EPERM; - } - break; - case I915_EXEC_BLT: - ring = &dev_priv->ring[BCS]; - if (ctx_id != DEFAULT_CONTEXT_ID) { - DRM_DEBUG("Ring %s doesn't support contexts\n", - ring->name); - return -EPERM; - } - break; - case I915_EXEC_VEBOX: - ring = &dev_priv->ring[VECS]; - if (ctx_id != DEFAULT_CONTEXT_ID) { - DRM_DEBUG("Ring %s doesn't support contexts\n", - ring->name); - return -EPERM; - } - break; - - default: + if ((args->flags & I915_EXEC_RING_MASK) > LAST_USER_RING) { DRM_DEBUG("execbuf with unknown ring: %d\n", (int)(args->flags & I915_EXEC_RING_MASK)); return -EINVAL; } + + if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT) + ring = &dev_priv->ring[RCS]; + else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) { + if (HAS_BSD2(dev)) { + int ring_id; + ring_id = gen8_dispatch_bsd_ring(dev, file); + ring = &dev_priv->ring[ring_id]; + } else + ring = &dev_priv->ring[VCS]; + } else + ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1]; + if (!intel_ring_initialized(ring)) { DRM_DEBUG("execbuf with invalid ring: %d\n", (int)(args->flags & I915_EXEC_RING_MASK)); @@ -1067,14 +1144,22 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, case I915_EXEC_CONSTANTS_REL_GENERAL: case I915_EXEC_CONSTANTS_ABSOLUTE: case I915_EXEC_CONSTANTS_REL_SURFACE: - if (ring == &dev_priv->ring[RCS] && - mode != dev_priv->relative_constants_mode) { - if (INTEL_INFO(dev)->gen < 4) + if (mode != 0 && ring != &dev_priv->ring[RCS]) { + DRM_DEBUG("non-0 rel constants mode on non-RCS\n"); + return -EINVAL; + } + + if (mode != dev_priv->relative_constants_mode) { + if (INTEL_INFO(dev)->gen < 4) { + DRM_DEBUG("no rel constants on pre-gen4\n"); return -EINVAL; + } if (INTEL_INFO(dev)->gen > 5 && - mode == I915_EXEC_CONSTANTS_REL_SURFACE) + mode == I915_EXEC_CONSTANTS_REL_SURFACE) { + DRM_DEBUG("rel surface constants mode invalid on gen5+\n"); return -EINVAL; + } /* The HW changed the meaning on this bit on gen6 */ if (INTEL_INFO(dev)->gen >= 6) @@ -1122,6 +1207,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, ret = -EFAULT; goto pre_mutex_err; } + } else { + if (args->DR4 == 0xffffffff) { + DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); + args->DR4 = 0; + } + + if (args->DR1 || args->DR4 || args->cliprects_ptr) { + DRM_DEBUG("0 cliprects but dirt in cliprects fields\n"); + return -EINVAL; + } } intel_runtime_pm_get(dev_priv); @@ -1136,14 +1231,22 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, goto pre_mutex_err; } - ret = i915_gem_validate_context(dev, file, ctx_id); - if (ret) { + ctx = i915_gem_validate_context(dev, file, ring, ctx_id); + if (IS_ERR(ctx)) { mutex_unlock(&dev->struct_mutex); + ret = PTR_ERR(ctx); goto pre_mutex_err; } + i915_gem_context_reference(ctx); + + vm = ctx->vm; + if (!USES_FULL_PPGTT(dev)) + vm = &dev_priv->gtt.base; + eb = eb_create(args); if (eb == NULL) { + i915_gem_context_unreference(ctx); mutex_unlock(&dev->struct_mutex); ret = -ENOMEM; goto pre_mutex_err; @@ -1155,7 +1258,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, goto err; /* take note of the batch buffer before we might reorder the lists */ - batch_obj = list_entry(eb->vmas.prev, struct i915_vma, exec_list)->obj; + batch_obj = eb_get_batch(eb); /* Move the objects en-masse into the GTT, evicting if necessary. */ need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0; @@ -1184,17 +1287,46 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, } batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; + if (i915_needs_cmd_parser(ring)) { + ret = i915_parse_cmds(ring, + batch_obj, + args->batch_start_offset, + file->is_master); + if (ret) + goto err; + + /* + * XXX: Actually do this when enabling batch copy... + * + * Set the DISPATCH_SECURE bit to remove the NON_SECURE bit + * from MI_BATCH_BUFFER_START commands issued in the + * dispatch_execbuffer implementations. We specifically don't + * want that set when the command parser is enabled. + */ + } + /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure * batch" bit. Hence we need to pin secure batches into the global gtt. * hsw should have this fixed, but bdw mucks it up again. */ - if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping) - i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level); + if (flags & I915_DISPATCH_SECURE && + !batch_obj->has_global_gtt_mapping) { + /* When we have multiple VMs, we'll need to make sure that we + * allocate space first */ + struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj); + BUG_ON(!vma); + vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND); + } + + if (flags & I915_DISPATCH_SECURE) + exec_start += i915_gem_obj_ggtt_offset(batch_obj); + else + exec_start += i915_gem_obj_offset(batch_obj, vm); ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->vmas); if (ret) goto err; - ret = i915_switch_context(ring, file, ctx_id); + ret = i915_switch_context(ring, ctx); if (ret) goto err; @@ -1219,8 +1351,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, goto err; } - exec_start = i915_gem_obj_offset(batch_obj, vm) + - args->batch_start_offset; + exec_len = args->batch_len; if (cliprects) { for (i = 0; i < args->num_cliprects; i++) { @@ -1249,6 +1380,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); err: + /* the request owns the ref now */ + i915_gem_context_unreference(ctx); eb_destroy(eb); mutex_unlock(&dev->struct_mutex); @@ -1270,7 +1403,6 @@ int i915_gem_execbuffer(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_execbuffer *args = data; struct drm_i915_gem_execbuffer2 exec2; struct drm_i915_gem_exec_object *exec_list = NULL; @@ -1326,21 +1458,23 @@ i915_gem_execbuffer(struct drm_device *dev, void *data, exec2.flags = I915_EXEC_RENDER; i915_execbuffer2_set_context_id(exec2, 0); - ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list, - &dev_priv->gtt.base); + ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); if (!ret) { + struct drm_i915_gem_exec_object __user *user_exec_list = + to_user_ptr(args->buffers_ptr); + /* Copy the new buffer offsets back to the user's exec list. */ - for (i = 0; i < args->buffer_count; i++) - exec_list[i].offset = exec2_list[i].offset; - /* ... and back out to userspace */ - ret = copy_to_user(to_user_ptr(args->buffers_ptr), - exec_list, - sizeof(*exec_list) * args->buffer_count); - if (ret) { - ret = -EFAULT; - DRM_DEBUG("failed to copy %d exec entries " - "back to user (%d)\n", - args->buffer_count, ret); + for (i = 0; i < args->buffer_count; i++) { + ret = __copy_to_user(&user_exec_list[i].offset, + &exec2_list[i].offset, + sizeof(user_exec_list[i].offset)); + if (ret) { + ret = -EFAULT; + DRM_DEBUG("failed to copy %d exec entries " + "back to user (%d)\n", + args->buffer_count, ret); + break; + } } } @@ -1353,7 +1487,6 @@ int i915_gem_execbuffer2(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_execbuffer2 *args = data; struct drm_i915_gem_exec_object2 *exec2_list = NULL; int ret; @@ -1364,6 +1497,11 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data, return -EINVAL; } + if (args->rsvd2 != 0) { + DRM_DEBUG("dirty rvsd2 field\n"); + return -EINVAL; + } + exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); if (exec2_list == NULL) @@ -1384,18 +1522,24 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data, return -EFAULT; } - ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list, - &dev_priv->gtt.base); + ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); if (!ret) { /* Copy the new buffer offsets back to the user's exec list. */ - ret = copy_to_user(to_user_ptr(args->buffers_ptr), - exec2_list, - sizeof(*exec2_list) * args->buffer_count); - if (ret) { - ret = -EFAULT; - DRM_DEBUG("failed to copy %d exec entries " - "back to user (%d)\n", - args->buffer_count, ret); + struct drm_i915_gem_exec_object2 *user_exec_list = + to_user_ptr(args->buffers_ptr); + int i; + + for (i = 0; i < args->buffer_count; i++) { + ret = __copy_to_user(&user_exec_list[i].offset, + &exec2_list[i].offset, + sizeof(user_exec_list[i].offset)); + if (ret) { + ret = -EFAULT; + DRM_DEBUG("failed to copy %d exec entries " + "back to user\n", + args->buffer_count); + break; + } } } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 40a2b36b276..8b3cde70336 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -1,5 +1,6 @@ /* * Copyright © 2010 Daniel Vetter + * Copyright © 2011-2014 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -22,53 +23,55 @@ * */ +#include <linux/seq_file.h> #include <drm/drmP.h> #include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" #include "intel_drv.h" -#define GEN6_PPGTT_PD_ENTRIES 512 -#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t)) -typedef uint64_t gen8_gtt_pte_t; -typedef gen8_gtt_pte_t gen8_ppgtt_pde_t; - -/* PPGTT stuff */ -#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) -#define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) - -#define GEN6_PDE_VALID (1 << 0) -/* gen6+ has bit 11-4 for physical addr bit 39-32 */ -#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) - -#define GEN6_PTE_VALID (1 << 0) -#define GEN6_PTE_UNCACHED (1 << 1) -#define HSW_PTE_UNCACHED (0) -#define GEN6_PTE_CACHE_LLC (2 << 1) -#define GEN7_PTE_CACHE_L3_LLC (3 << 1) -#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) -#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) - -/* Cacheability Control is a 4-bit value. The low three bits are stored in * - * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. - */ -#define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ - (((bits) & 0x8) << (11 - 3))) -#define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) -#define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) -#define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) -#define HSW_WB_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x8) -#define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) -#define HSW_WT_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x7) - -#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t)) -#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t)) -#define GEN8_LEGACY_PDPS 4 - -#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD) -#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */ -#define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */ -#define PPAT_DISPLAY_ELLC_INDEX _PAGE_PCD /* WT eLLC */ +static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv); +static void chv_setup_private_ppat(struct drm_i915_private *dev_priv); + +bool intel_enable_ppgtt(struct drm_device *dev, bool full) +{ + if (i915.enable_ppgtt == 0) + return false; + + if (i915.enable_ppgtt == 1 && full) + return false; + + return true; +} + +static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) +{ + if (enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev)) + return 0; + + if (enable_ppgtt == 1) + return 1; + + if (enable_ppgtt == 2 && HAS_PPGTT(dev)) + return 2; + +#ifdef CONFIG_INTEL_IOMMU + /* Disable ppgtt on SNB if VT-d is on. */ + if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { + DRM_INFO("Disabling PPGTT because VT-d is on\n"); + return 0; + } +#endif + + return HAS_ALIASING_PPGTT(dev) ? 1 : 0; +} + + +static void ppgtt_bind_vma(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags); +static void ppgtt_unbind_vma(struct i915_vma *vma); +static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt); static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr, enum i915_cache_level level, @@ -76,10 +79,19 @@ static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr, { gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; pte |= addr; - if (level != I915_CACHE_NONE) - pte |= PPAT_CACHED_INDEX; - else + + switch (level) { + case I915_CACHE_NONE: pte |= PPAT_UNCACHED_INDEX; + break; + case I915_CACHE_WT: + pte |= PPAT_DISPLAY_ELLC_INDEX; + break; + default: + pte |= PPAT_CACHED_INDEX; + break; + } + return pte; } @@ -142,9 +154,6 @@ static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr, return pte; } -#define BYT_PTE_WRITEABLE (1 << 1) -#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) - static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr, enum i915_cache_level level, bool valid) @@ -198,13 +207,20 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, } /* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry, - uint64_t val) +static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry, + uint64_t val, bool synchronous) { + struct drm_i915_private *dev_priv = ring->dev->dev_private; int ret; BUG_ON(entry >= 4); + if (synchronous) { + I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32); + I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val); + return 0; + } + ret = intel_ring_begin(ring, 6); if (ret) return ret; @@ -220,216 +236,364 @@ static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry, return 0; } -static int gen8_ppgtt_enable(struct drm_device *dev) +static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *ring, + bool synchronous) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - int i, j, ret; + int i, ret; /* bit of a hack to find the actual last used pd */ int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE; - for_each_ring(ring, dev_priv, j) { - I915_WRITE(RING_MODE_GEN7(ring), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - } - for (i = used_pd - 1; i >= 0; i--) { dma_addr_t addr = ppgtt->pd_dma_addr[i]; - for_each_ring(ring, dev_priv, j) { - ret = gen8_write_pdp(ring, i, addr); - if (ret) - goto err_out; - } + ret = gen8_write_pdp(ring, i, addr, synchronous); + if (ret) + return ret; } - return 0; -err_out: - for_each_ring(ring, dev_priv, j) - I915_WRITE(RING_MODE_GEN7(ring), - _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE)); - return ret; + return 0; } static void gen8_ppgtt_clear_range(struct i915_address_space *vm, - unsigned first_entry, - unsigned num_entries, + uint64_t start, + uint64_t length, bool use_scratch) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); gen8_gtt_pte_t *pt_vaddr, scratch_pte; - unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE; - unsigned first_pte = first_entry % GEN8_PTES_PER_PAGE; + unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; + unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; + unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; + unsigned num_entries = length >> PAGE_SHIFT; unsigned last_pte, i; scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr, I915_CACHE_LLC, use_scratch); while (num_entries) { - struct page *page_table = &ppgtt->gen8_pt_pages[act_pt]; + struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; - last_pte = first_pte + num_entries; + last_pte = pte + num_entries; if (last_pte > GEN8_PTES_PER_PAGE) last_pte = GEN8_PTES_PER_PAGE; pt_vaddr = kmap_atomic(page_table); - for (i = first_pte; i < last_pte; i++) + for (i = pte; i < last_pte; i++) { pt_vaddr[i] = scratch_pte; + num_entries--; + } + if (!HAS_LLC(ppgtt->base.dev)) + drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); kunmap_atomic(pt_vaddr); - num_entries -= last_pte - first_pte; - first_pte = 0; - act_pt++; + pte = 0; + if (++pde == GEN8_PDES_PER_PAGE) { + pdpe++; + pde = 0; + } } } static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, struct sg_table *pages, - unsigned first_entry, + uint64_t start, enum i915_cache_level cache_level) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); gen8_gtt_pte_t *pt_vaddr; - unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE; - unsigned act_pte = first_entry % GEN8_PTES_PER_PAGE; + unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; + unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; + unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; struct sg_page_iter sg_iter; pt_vaddr = NULL; + for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { + if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS)) + break; + if (pt_vaddr == NULL) - pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]); + pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]); - pt_vaddr[act_pte] = + pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter), cache_level, true); - if (++act_pte == GEN8_PTES_PER_PAGE) { + if (++pte == GEN8_PTES_PER_PAGE) { + if (!HAS_LLC(ppgtt->base.dev)) + drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); kunmap_atomic(pt_vaddr); pt_vaddr = NULL; - act_pt++; - act_pte = 0; + if (++pde == GEN8_PDES_PER_PAGE) { + pdpe++; + pde = 0; + } + pte = 0; } } - if (pt_vaddr) + if (pt_vaddr) { + if (!HAS_LLC(ppgtt->base.dev)) + drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); kunmap_atomic(pt_vaddr); + } +} + +static void gen8_free_page_tables(struct page **pt_pages) +{ + int i; + + if (pt_pages == NULL) + return; + + for (i = 0; i < GEN8_PDES_PER_PAGE; i++) + if (pt_pages[i]) + __free_pages(pt_pages[i], 0); +} + +static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt) +{ + int i; + + for (i = 0; i < ppgtt->num_pd_pages; i++) { + gen8_free_page_tables(ppgtt->gen8_pt_pages[i]); + kfree(ppgtt->gen8_pt_pages[i]); + kfree(ppgtt->gen8_pt_dma_addr[i]); + } + + __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT)); +} + +static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) +{ + struct pci_dev *hwdev = ppgtt->base.dev->pdev; + int i, j; + + for (i = 0; i < ppgtt->num_pd_pages; i++) { + /* TODO: In the future we'll support sparse mappings, so this + * will have to change. */ + if (!ppgtt->pd_dma_addr[i]) + continue; + + pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE, + PCI_DMA_BIDIRECTIONAL); + + for (j = 0; j < GEN8_PDES_PER_PAGE; j++) { + dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; + if (addr) + pci_unmap_page(hwdev, addr, PAGE_SIZE, + PCI_DMA_BIDIRECTIONAL); + } + } } static void gen8_ppgtt_cleanup(struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); - int i, j; + list_del(&vm->global_link); drm_mm_takedown(&vm->mm); - for (i = 0; i < ppgtt->num_pd_pages ; i++) { - if (ppgtt->pd_dma_addr[i]) { - pci_unmap_page(ppgtt->base.dev->pdev, - ppgtt->pd_dma_addr[i], - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + gen8_ppgtt_unmap_pages(ppgtt); + gen8_ppgtt_free(ppgtt); +} + +static struct page **__gen8_alloc_page_tables(void) +{ + struct page **pt_pages; + int i; - for (j = 0; j < GEN8_PDES_PER_PAGE; j++) { - dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; - if (addr) - pci_unmap_page(ppgtt->base.dev->pdev, - addr, - PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); + pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL); + if (!pt_pages) + return ERR_PTR(-ENOMEM); - } - } - kfree(ppgtt->gen8_pt_dma_addr[i]); + for (i = 0; i < GEN8_PDES_PER_PAGE; i++) { + pt_pages[i] = alloc_page(GFP_KERNEL); + if (!pt_pages[i]) + goto bail; } - __free_pages(ppgtt->gen8_pt_pages, get_order(ppgtt->num_pt_pages << PAGE_SHIFT)); - __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT)); + return pt_pages; + +bail: + gen8_free_page_tables(pt_pages); + kfree(pt_pages); + return ERR_PTR(-ENOMEM); } -/** - * GEN8 legacy ppgtt programming is accomplished through 4 PDP registers with a - * net effect resembling a 2-level page table in normal x86 terms. Each PDP - * represents 1GB of memory - * 4 * 512 * 512 * 4096 = 4GB legacy 32b address space. - * - * TODO: Do something with the size parameter - **/ -static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) +static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt, + const int max_pdp) { - struct page *pt_pages; - int i, j, ret = -ENOMEM; - const int max_pdp = DIV_ROUND_UP(size, 1 << 30); - const int num_pt_pages = GEN8_PDES_PER_PAGE * max_pdp; + struct page **pt_pages[GEN8_LEGACY_PDPS]; + int i, ret; - if (size % (1<<30)) - DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size); + for (i = 0; i < max_pdp; i++) { + pt_pages[i] = __gen8_alloc_page_tables(); + if (IS_ERR(pt_pages[i])) { + ret = PTR_ERR(pt_pages[i]); + goto unwind_out; + } + } - /* FIXME: split allocation into smaller pieces. For now we only ever do - * this once, but with full PPGTT, the multiple contiguous allocations - * will be bad. + /* NB: Avoid touching gen8_pt_pages until last to keep the allocation, + * "atomic" - for cleanup purposes. */ + for (i = 0; i < max_pdp; i++) + ppgtt->gen8_pt_pages[i] = pt_pages[i]; + + return 0; + +unwind_out: + while (i--) { + gen8_free_page_tables(pt_pages[i]); + kfree(pt_pages[i]); + } + + return ret; +} + +static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt) +{ + int i; + + for (i = 0; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE, + sizeof(dma_addr_t), + GFP_KERNEL); + if (!ppgtt->gen8_pt_dma_addr[i]) + return -ENOMEM; + } + + return 0; +} + +static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt, + const int max_pdp) +{ ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT)); if (!ppgtt->pd_pages) return -ENOMEM; - pt_pages = alloc_pages(GFP_KERNEL, get_order(num_pt_pages << PAGE_SHIFT)); - if (!pt_pages) { + ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT); + BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS); + + return 0; +} + +static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, + const int max_pdp) +{ + int ret; + + ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp); + if (ret) + return ret; + + ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp); + if (ret) { __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT)); - return -ENOMEM; + return ret; } - ppgtt->gen8_pt_pages = pt_pages; - ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT); - ppgtt->num_pt_pages = 1 << get_order(num_pt_pages << PAGE_SHIFT); ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE; - ppgtt->enable = gen8_ppgtt_enable; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.start = 0; - ppgtt->base.total = ppgtt->num_pt_pages * GEN8_PTES_PER_PAGE * PAGE_SIZE; - BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS); + ret = gen8_ppgtt_allocate_dma(ppgtt); + if (ret) + gen8_ppgtt_free(ppgtt); - /* - * - Create a mapping for the page directories. - * - For each page directory: - * allocate space for page table mappings. - * map each page table - */ - for (i = 0; i < max_pdp; i++) { - dma_addr_t temp; - temp = pci_map_page(ppgtt->base.dev->pdev, - &ppgtt->pd_pages[i], 0, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp)) - goto err_out; + return ret; +} - ppgtt->pd_dma_addr[i] = temp; +static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt, + const int pd) +{ + dma_addr_t pd_addr; + int ret; - ppgtt->gen8_pt_dma_addr[i] = kmalloc(sizeof(dma_addr_t) * GEN8_PDES_PER_PAGE, GFP_KERNEL); - if (!ppgtt->gen8_pt_dma_addr[i]) - goto err_out; + pd_addr = pci_map_page(ppgtt->base.dev->pdev, + &ppgtt->pd_pages[pd], 0, + PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - for (j = 0; j < GEN8_PDES_PER_PAGE; j++) { - struct page *p = &pt_pages[i * GEN8_PDES_PER_PAGE + j]; - temp = pci_map_page(ppgtt->base.dev->pdev, - p, 0, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); + ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr); + if (ret) + return ret; - if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp)) - goto err_out; + ppgtt->pd_dma_addr[pd] = pd_addr; - ppgtt->gen8_pt_dma_addr[i][j] = temp; + return 0; +} + +static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, + const int pd, + const int pt) +{ + dma_addr_t pt_addr; + struct page *p; + int ret; + + p = ppgtt->gen8_pt_pages[pd][pt]; + pt_addr = pci_map_page(ppgtt->base.dev->pdev, + p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr); + if (ret) + return ret; + + ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr; + + return 0; +} + +/** + * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers + * with a net effect resembling a 2-level page table in normal x86 terms. Each + * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address + * space. + * + * FIXME: split allocation into smaller pieces. For now we only ever do this + * once, but with full PPGTT, the multiple contiguous allocations will be bad. + * TODO: Do something with the size parameter + */ +static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) +{ + const int max_pdp = DIV_ROUND_UP(size, 1 << 30); + const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp; + int i, j, ret; + + if (size % (1<<30)) + DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size); + + /* 1. Do all our allocations for page directories and page tables. */ + ret = gen8_ppgtt_alloc(ppgtt, max_pdp); + if (ret) + return ret; + + /* + * 2. Create DMA mappings for the page directories and page tables. + */ + for (i = 0; i < max_pdp; i++) { + ret = gen8_ppgtt_setup_page_directories(ppgtt, i); + if (ret) + goto bail; + + for (j = 0; j < GEN8_PDES_PER_PAGE; j++) { + ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j); + if (ret) + goto bail; } } - /* For now, the PPGTT helper functions all require that the PDEs are + /* + * 3. Map all the page directory entires to point to the page tables + * we've allocated. + * + * For now, the PPGTT helper functions all require that the PDEs are * plugged in correctly. So we do that now/here. For aliasing PPGTT, we - * will never need to touch the PDEs again */ + * will never need to touch the PDEs again. + */ for (i = 0; i < max_pdp; i++) { gen8_ppgtt_pde_t *pd_vaddr; pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]); @@ -438,26 +602,90 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr, I915_CACHE_LLC); } + if (!HAS_LLC(ppgtt->base.dev)) + drm_clflush_virt_range(pd_vaddr, PAGE_SIZE); kunmap_atomic(pd_vaddr); } - ppgtt->base.clear_range(&ppgtt->base, 0, - ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE, - true); + ppgtt->enable = gen8_ppgtt_enable; + ppgtt->switch_mm = gen8_mm_switch; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.start = 0; + ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE; + + ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true); DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n", ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp); DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n", - ppgtt->num_pt_pages, - (ppgtt->num_pt_pages - num_pt_pages) + - size % (1<<30)); + ppgtt->num_pd_entries, + (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30)); return 0; -err_out: - ppgtt->base.cleanup(&ppgtt->base); +bail: + gen8_ppgtt_unmap_pages(ppgtt); + gen8_ppgtt_free(ppgtt); return ret; } +static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) +{ + struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; + struct i915_address_space *vm = &ppgtt->base; + gen6_gtt_pte_t __iomem *pd_addr; + gen6_gtt_pte_t scratch_pte; + uint32_t pd_entry; + int pte, pde; + + scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true); + + pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); + + seq_printf(m, " VM %p (pd_offset %x-%x):\n", vm, + ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries); + for (pde = 0; pde < ppgtt->num_pd_entries; pde++) { + u32 expected; + gen6_gtt_pte_t *pt_vaddr; + dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde]; + pd_entry = readl(pd_addr + pde); + expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); + + if (pd_entry != expected) + seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", + pde, + pd_entry, + expected); + seq_printf(m, "\tPDE: %x\n", pd_entry); + + pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]); + for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) { + unsigned long va = + (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) + + (pte * PAGE_SIZE); + int i; + bool found = false; + for (i = 0; i < 4; i++) + if (pt_vaddr[pte + i] != scratch_pte) + found = true; + if (!found) + continue; + + seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte); + for (i = 0; i < 4; i++) { + if (pt_vaddr[pte + i] != scratch_pte) + seq_printf(m, " %08x", pt_vaddr[pte + i]); + else + seq_puts(m, " SCRATCH "); + } + seq_puts(m, "\n"); + } + kunmap_atomic(pt_vaddr); + } +} + static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt) { struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; @@ -480,73 +708,235 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt) readl(pd_addr); } -static int gen6_ppgtt_enable(struct drm_device *dev) +static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) { - drm_i915_private_t *dev_priv = dev->dev_private; - uint32_t pd_offset; - struct intel_ring_buffer *ring; - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - int i; - BUG_ON(ppgtt->pd_offset & 0x3f); - gen6_write_pdes(ppgtt); + return (ppgtt->pd_offset / 64) << 16; +} + +static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *ring, + bool synchronous) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + /* If we're in reset, we can assume the GPU is sufficiently idle to + * manually frob these bits. Ideally we could use the ring functions, + * except our error handling makes it quite difficult (can't use + * intel_ring_begin, ring->flush, or intel_ring_advance) + * + * FIXME: We should try not to special case reset + */ + if (synchronous || + i915_reset_in_progress(&dev_priv->gpu_error)) { + WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt); + I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); + POSTING_READ(RING_PP_DIR_BASE(ring)); + return 0; + } + + /* NB: TLBs must be flushed and invalidated before a switch */ + ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + if (ret) + return ret; - pd_offset = ppgtt->pd_offset; - pd_offset /= 64; /* in cachelines, */ - pd_offset <<= 16; + ret = intel_ring_begin(ring, 6); + if (ret) + return ret; - if (INTEL_INFO(dev)->gen == 6) { - uint32_t ecochk, gab_ctl, ecobits; + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); + intel_ring_emit(ring, PP_DIR_DCLV_2G); + intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); + intel_ring_emit(ring, get_pd_offset(ppgtt)); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); - ecobits = I915_READ(GAC_ECO_BITS); - I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | - ECOBITS_PPGTT_CACHE64B); + return 0; +} - gab_ctl = I915_READ(GAB_CTL); - I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); +static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *ring, + bool synchronous) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; - ecochk = I915_READ(GAM_ECOCHK); - I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | - ECOCHK_PPGTT_CACHE64B); - I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - } else if (INTEL_INFO(dev)->gen >= 7) { - uint32_t ecochk, ecobits; + /* If we're in reset, we can assume the GPU is sufficiently idle to + * manually frob these bits. Ideally we could use the ring functions, + * except our error handling makes it quite difficult (can't use + * intel_ring_begin, ring->flush, or intel_ring_advance) + * + * FIXME: We should try not to special case reset + */ + if (synchronous || + i915_reset_in_progress(&dev_priv->gpu_error)) { + WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt); + I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); + POSTING_READ(RING_PP_DIR_BASE(ring)); + return 0; + } - ecobits = I915_READ(GAC_ECO_BITS); - I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); + /* NB: TLBs must be flushed and invalidated before a switch */ + ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + if (ret) + return ret; - ecochk = I915_READ(GAM_ECOCHK); - if (IS_HASWELL(dev)) { - ecochk |= ECOCHK_PPGTT_WB_HSW; - } else { - ecochk |= ECOCHK_PPGTT_LLC_IVB; - ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; - } - I915_WRITE(GAM_ECOCHK, ecochk); - /* GFX_MODE is per-ring on gen7+ */ + ret = intel_ring_begin(ring, 6); + if (ret) + return ret; + + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit(ring, RING_PP_DIR_DCLV(ring)); + intel_ring_emit(ring, PP_DIR_DCLV_2G); + intel_ring_emit(ring, RING_PP_DIR_BASE(ring)); + intel_ring_emit(ring, get_pd_offset(ppgtt)); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); + + /* XXX: RCS is the only one to auto invalidate the TLBs? */ + if (ring->id != RCS) { + ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + if (ret) + return ret; + } + + return 0; +} + +static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *ring, + bool synchronous) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + if (!synchronous) + return 0; + + I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); + + POSTING_READ(RING_PP_DIR_DCLV(ring)); + + return 0; +} + +static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; + int j, ret; + + for_each_ring(ring, dev_priv, j) { + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); + + /* We promise to do a switch later with FULL PPGTT. If this is + * aliasing, this is the one and only switch we'll do */ + if (USES_FULL_PPGTT(dev)) + continue; + + ret = ppgtt->switch_mm(ppgtt, ring, true); + if (ret) + goto err_out; } + return 0; + +err_out: + for_each_ring(ring, dev_priv, j) + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE)); + return ret; +} + +static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; + uint32_t ecochk, ecobits; + int i; + + ecobits = I915_READ(GAC_ECO_BITS); + I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); + + ecochk = I915_READ(GAM_ECOCHK); + if (IS_HASWELL(dev)) { + ecochk |= ECOCHK_PPGTT_WB_HSW; + } else { + ecochk |= ECOCHK_PPGTT_LLC_IVB; + ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; + } + I915_WRITE(GAM_ECOCHK, ecochk); + for_each_ring(ring, dev_priv, i) { - if (INTEL_INFO(dev)->gen >= 7) - I915_WRITE(RING_MODE_GEN7(ring), - _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); + int ret; + /* GFX_MODE is per-ring on gen7+ */ + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); + /* We promise to do a switch later with FULL PPGTT. If this is + * aliasing, this is the one and only switch we'll do */ + if (USES_FULL_PPGTT(dev)) + continue; + + ret = ppgtt->switch_mm(ppgtt, ring, true); + if (ret) + return ret; } + + return 0; +} + +static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; + uint32_t ecochk, gab_ctl, ecobits; + int i; + + ecobits = I915_READ(GAC_ECO_BITS); + I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | + ECOBITS_PPGTT_CACHE64B); + + gab_ctl = I915_READ(GAB_CTL); + I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); + + ecochk = I915_READ(GAM_ECOCHK); + I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B); + + I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); + + for_each_ring(ring, dev_priv, i) { + int ret = ppgtt->switch_mm(ppgtt, ring, true); + if (ret) + return ret; + } + return 0; } /* PPGTT support for Sandybdrige/Gen6 and later */ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, - unsigned first_entry, - unsigned num_entries, + uint64_t start, + uint64_t length, bool use_scratch) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); gen6_gtt_pte_t *pt_vaddr, scratch_pte; + unsigned first_entry = start >> PAGE_SHIFT; + unsigned num_entries = length >> PAGE_SHIFT; unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; unsigned last_pte, i; @@ -573,12 +963,13 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm, static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, struct sg_table *pages, - unsigned first_entry, + uint64_t start, enum i915_cache_level cache_level) { struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); gen6_gtt_pte_t *pt_vaddr; + unsigned first_entry = start >> PAGE_SHIFT; unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; struct sg_page_iter sg_iter; @@ -602,65 +993,129 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, kunmap_atomic(pt_vaddr); } -static void gen6_ppgtt_cleanup(struct i915_address_space *vm) +static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) { - struct i915_hw_ppgtt *ppgtt = - container_of(vm, struct i915_hw_ppgtt, base); int i; - drm_mm_takedown(&ppgtt->base.mm); - if (ppgtt->pt_dma_addr) { for (i = 0; i < ppgtt->num_pd_entries; i++) pci_unmap_page(ppgtt->base.dev->pdev, ppgtt->pt_dma_addr[i], 4096, PCI_DMA_BIDIRECTIONAL); } +} + +static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) +{ + int i; kfree(ppgtt->pt_dma_addr); for (i = 0; i < ppgtt->num_pd_entries; i++) __free_page(ppgtt->pt_pages[i]); kfree(ppgtt->pt_pages); - kfree(ppgtt); } -static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) +static void gen6_ppgtt_cleanup(struct i915_address_space *vm) +{ + struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + + list_del(&vm->global_link); + drm_mm_takedown(&ppgtt->base.mm); + drm_mm_remove_node(&ppgtt->node); + + gen6_ppgtt_unmap_pages(ppgtt); + gen6_ppgtt_free(ppgtt); +} + +static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) { struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - unsigned first_pd_entry_in_global_pt; - int i; - int ret = -ENOMEM; + bool retried = false; + int ret; - /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 - * entries. For aliasing ppgtt support we just steal them at the end for - * now. */ - first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); + /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The + * allocator works in address space sizes, so it's multiplied by page + * size. We allocate at the top of the GTT to avoid fragmentation. + */ + BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); +alloc: + ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, + &ppgtt->node, GEN6_PD_SIZE, + GEN6_PD_ALIGN, 0, + 0, dev_priv->gtt.base.total, + DRM_MM_TOPDOWN); + if (ret == -ENOSPC && !retried) { + ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, + GEN6_PD_SIZE, GEN6_PD_ALIGN, + I915_CACHE_NONE, + 0, dev_priv->gtt.base.total, + 0); + if (ret) + return ret; + + retried = true; + goto alloc; + } + + if (ppgtt->node.start < dev_priv->gtt.mappable_end) + DRM_DEBUG("Forced to use aperture for PDEs\n"); - ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES; - ppgtt->enable = gen6_ppgtt_enable; - ppgtt->base.clear_range = gen6_ppgtt_clear_range; - ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; - ppgtt->base.cleanup = gen6_ppgtt_cleanup; - ppgtt->base.scratch = dev_priv->gtt.base.scratch; - ppgtt->base.start = 0; - ppgtt->base.total = GEN6_PPGTT_PD_ENTRIES * I915_PPGTT_PT_ENTRIES * PAGE_SIZE; + return ret; +} + +static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) +{ + int i; + ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *), GFP_KERNEL); + if (!ppgtt->pt_pages) return -ENOMEM; for (i = 0; i < ppgtt->num_pd_entries; i++) { ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); - if (!ppgtt->pt_pages[i]) - goto err_pt_alloc; + if (!ppgtt->pt_pages[i]) { + gen6_ppgtt_free(ppgtt); + return -ENOMEM; + } + } + + return 0; +} + +static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) +{ + int ret; + + ret = gen6_ppgtt_allocate_page_directories(ppgtt); + if (ret) + return ret; + + ret = gen6_ppgtt_allocate_page_tables(ppgtt); + if (ret) { + drm_mm_remove_node(&ppgtt->node); + return ret; } ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t), GFP_KERNEL); - if (!ppgtt->pt_dma_addr) - goto err_pt_alloc; + if (!ppgtt->pt_dma_addr) { + drm_mm_remove_node(&ppgtt->node); + gen6_ppgtt_free(ppgtt); + return -ENOMEM; + } + + return 0; +} + +static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt) +{ + struct drm_device *dev = ppgtt->base.dev; + int i; for (i = 0; i < ppgtt->num_pd_entries; i++) { dma_addr_t pt_addr; @@ -669,48 +1124,71 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) PCI_DMA_BIDIRECTIONAL); if (pci_dma_mapping_error(dev->pdev, pt_addr)) { - ret = -EIO; - goto err_pd_pin; - + gen6_ppgtt_unmap_pages(ppgtt); + return -EIO; } + ppgtt->pt_dma_addr[i] = pt_addr; } - ppgtt->base.clear_range(&ppgtt->base, 0, - ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true); + return 0; +} - ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); +static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) +{ + struct drm_device *dev = ppgtt->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; - return 0; + ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; + if (IS_GEN6(dev)) { + ppgtt->enable = gen6_ppgtt_enable; + ppgtt->switch_mm = gen6_mm_switch; + } else if (IS_HASWELL(dev)) { + ppgtt->enable = gen7_ppgtt_enable; + ppgtt->switch_mm = hsw_mm_switch; + } else if (IS_GEN7(dev)) { + ppgtt->enable = gen7_ppgtt_enable; + ppgtt->switch_mm = gen7_mm_switch; + } else + BUG(); -err_pd_pin: - if (ppgtt->pt_dma_addr) { - for (i--; i >= 0; i--) - pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], - 4096, PCI_DMA_BIDIRECTIONAL); - } -err_pt_alloc: - kfree(ppgtt->pt_dma_addr); - for (i = 0; i < ppgtt->num_pd_entries; i++) { - if (ppgtt->pt_pages[i]) - __free_page(ppgtt->pt_pages[i]); + ret = gen6_ppgtt_alloc(ppgtt); + if (ret) + return ret; + + ret = gen6_ppgtt_setup_page_tables(ppgtt); + if (ret) { + gen6_ppgtt_free(ppgtt); + return ret; } - kfree(ppgtt->pt_pages); - return ret; + ppgtt->base.clear_range = gen6_ppgtt_clear_range; + ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; + ppgtt->base.cleanup = gen6_ppgtt_cleanup; + ppgtt->base.start = 0; + ppgtt->base.total = ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE; + ppgtt->debug_dump = gen6_dump_ppgtt; + + ppgtt->pd_offset = + ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t); + + ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true); + + DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n", + ppgtt->node.size >> 20, + ppgtt->node.start / PAGE_SIZE); + + return 0; } -static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) +int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) { struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_hw_ppgtt *ppgtt; - int ret; - - ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); - if (!ppgtt) - return -ENOMEM; + int ret = 0; ppgtt->base.dev = dev; + ppgtt->base.scratch = dev_priv->gtt.base.scratch; if (INTEL_INFO(dev)->gen < 8) ret = gen6_ppgtt_init(ppgtt); @@ -719,45 +1197,37 @@ static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) else BUG(); - if (ret) - kfree(ppgtt); - else { - dev_priv->mm.aliasing_ppgtt = ppgtt; + if (!ret) { + struct drm_i915_private *dev_priv = dev->dev_private; + kref_init(&ppgtt->ref); drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); + i915_init_vm(dev_priv, &ppgtt->base); + if (INTEL_INFO(dev)->gen < 8) { + gen6_write_pdes(ppgtt); + DRM_DEBUG("Adding PPGTT at offset %x\n", + ppgtt->pd_offset << 10); + } } return ret; } -void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) +static void +ppgtt_bind_vma(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - - if (!ppgtt) - return; - - ppgtt->base.cleanup(&ppgtt->base); - dev_priv->mm.aliasing_ppgtt = NULL; + vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, + cache_level); } -void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_object *obj, - enum i915_cache_level cache_level) +static void ppgtt_unbind_vma(struct i915_vma *vma) { - ppgtt->base.insert_entries(&ppgtt->base, obj->pages, - i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT, - cache_level); -} - -void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_object *obj) -{ - ppgtt->base.clear_range(&ppgtt->base, - i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT, - obj->base.size >> PAGE_SHIFT, - true); + vma->vm->clear_range(vma->vm, + vma->node.start, + vma->obj->base.size, + true); } extern int intel_iommu_gfx_mapped; @@ -801,7 +1271,7 @@ static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) void i915_check_and_clear_faults(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; int i; if (INTEL_INFO(dev)->gen < 6) @@ -840,27 +1310,59 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) i915_check_and_clear_faults(dev); dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, - dev_priv->gtt.base.start / PAGE_SIZE, - dev_priv->gtt.base.total / PAGE_SIZE, - false); + dev_priv->gtt.base.start, + dev_priv->gtt.base.total, + true); } void i915_gem_restore_gtt_mappings(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; + struct i915_address_space *vm; i915_check_and_clear_faults(dev); /* First fill our portion of the GTT with scratch pages */ dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, - dev_priv->gtt.base.start / PAGE_SIZE, - dev_priv->gtt.base.total / PAGE_SIZE, + dev_priv->gtt.base.start, + dev_priv->gtt.base.total, true); list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + struct i915_vma *vma = i915_gem_obj_to_vma(obj, + &dev_priv->gtt.base); + if (!vma) + continue; + i915_gem_clflush_object(obj, obj->pin_display); - i915_gem_gtt_bind_object(obj, obj->cache_level); + /* The bind_vma code tries to be smart about tracking mappings. + * Unfortunately above, we've just wiped out the mappings + * without telling our object about it. So we need to fake it. + */ + obj->has_global_gtt_mapping = 0; + vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND); + } + + + if (INTEL_INFO(dev)->gen >= 8) { + if (IS_CHERRYVIEW(dev)) + chv_setup_private_ppat(dev_priv); + else + bdw_setup_private_ppat(dev_priv); + + return; + } + + list_for_each_entry(vm, &dev_priv->vm_list, global_link) { + /* TODO: Perhaps it shouldn't be gen6 specific */ + if (i915_is_ggtt(vm)) { + if (dev_priv->mm.aliasing_ppgtt) + gen6_write_pdes(dev_priv->mm.aliasing_ppgtt); + continue; + } + + gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base)); } i915_gem_chipset_flush(dev); @@ -891,15 +1393,16 @@ static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte) static void gen8_ggtt_insert_entries(struct i915_address_space *vm, struct sg_table *st, - unsigned int first_entry, + uint64_t start, enum i915_cache_level level) { struct drm_i915_private *dev_priv = vm->dev->dev_private; + unsigned first_entry = start >> PAGE_SHIFT; gen8_gtt_pte_t __iomem *gtt_entries = (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; int i = 0; struct sg_page_iter sg_iter; - dma_addr_t addr; + dma_addr_t addr = 0; for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { addr = sg_dma_address(sg_iter.sg) + @@ -936,10 +1439,11 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, */ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, struct sg_table *st, - unsigned int first_entry, + uint64_t start, enum i915_cache_level level) { struct drm_i915_private *dev_priv = vm->dev->dev_private; + unsigned first_entry = start >> PAGE_SHIFT; gen6_gtt_pte_t __iomem *gtt_entries = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; int i = 0; @@ -971,11 +1475,13 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, } static void gen8_ggtt_clear_range(struct i915_address_space *vm, - unsigned int first_entry, - unsigned int num_entries, + uint64_t start, + uint64_t length, bool use_scratch) { struct drm_i915_private *dev_priv = vm->dev->dev_private; + unsigned first_entry = start >> PAGE_SHIFT; + unsigned num_entries = length >> PAGE_SHIFT; gen8_gtt_pte_t scratch_pte, __iomem *gtt_base = (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; @@ -995,11 +1501,13 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, } static void gen6_ggtt_clear_range(struct i915_address_space *vm, - unsigned int first_entry, - unsigned int num_entries, + uint64_t start, + uint64_t length, bool use_scratch) { struct drm_i915_private *dev_priv = vm->dev->dev_private; + unsigned first_entry = start >> PAGE_SHIFT; + unsigned num_entries = length >> PAGE_SHIFT; gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; @@ -1017,53 +1525,103 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, readl(gtt_base); } -static void i915_ggtt_insert_entries(struct i915_address_space *vm, - struct sg_table *st, - unsigned int pg_start, - enum i915_cache_level cache_level) + +static void i915_ggtt_bind_vma(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 unused) { + const unsigned long entry = vma->node.start >> PAGE_SHIFT; unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - intel_gtt_insert_sg_entries(st, pg_start, flags); - + BUG_ON(!i915_is_ggtt(vma->vm)); + intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags); + vma->obj->has_global_gtt_mapping = 1; } static void i915_ggtt_clear_range(struct i915_address_space *vm, - unsigned int first_entry, - unsigned int num_entries, + uint64_t start, + uint64_t length, bool unused) { + unsigned first_entry = start >> PAGE_SHIFT; + unsigned num_entries = length >> PAGE_SHIFT; intel_gtt_clear_range(first_entry, num_entries); } +static void i915_ggtt_unbind_vma(struct i915_vma *vma) +{ + const unsigned int first = vma->node.start >> PAGE_SHIFT; + const unsigned int size = vma->obj->base.size >> PAGE_SHIFT; + + BUG_ON(!i915_is_ggtt(vma->vm)); + vma->obj->has_global_gtt_mapping = 0; + intel_gtt_clear_range(first, size); +} -void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, - enum i915_cache_level cache_level) +static void ggtt_bind_vma(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags) { - struct drm_device *dev = obj->base.dev; + struct drm_device *dev = vma->vm->dev; struct drm_i915_private *dev_priv = dev->dev_private; - const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; + struct drm_i915_gem_object *obj = vma->obj; - dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages, - entry, - cache_level); + /* If there is no aliasing PPGTT, or the caller needs a global mapping, + * or we have a global mapping already but the cacheability flags have + * changed, set the global PTEs. + * + * If there is an aliasing PPGTT it is anecdotally faster, so use that + * instead if none of the above hold true. + * + * NB: A global mapping should only be needed for special regions like + * "gtt mappable", SNB errata, or if specified via special execbuf + * flags. At all other times, the GPU will use the aliasing PPGTT. + */ + if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) { + if (!obj->has_global_gtt_mapping || + (cache_level != obj->cache_level)) { + vma->vm->insert_entries(vma->vm, obj->pages, + vma->node.start, + cache_level); + obj->has_global_gtt_mapping = 1; + } + } - obj->has_global_gtt_mapping = 1; + if (dev_priv->mm.aliasing_ppgtt && + (!obj->has_aliasing_ppgtt_mapping || + (cache_level != obj->cache_level))) { + struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; + appgtt->base.insert_entries(&appgtt->base, + vma->obj->pages, + vma->node.start, + cache_level); + vma->obj->has_aliasing_ppgtt_mapping = 1; + } } -void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) +static void ggtt_unbind_vma(struct i915_vma *vma) { - struct drm_device *dev = obj->base.dev; + struct drm_device *dev = vma->vm->dev; struct drm_i915_private *dev_priv = dev->dev_private; - const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; - - dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, - entry, - obj->base.size >> PAGE_SHIFT, - true); + struct drm_i915_gem_object *obj = vma->obj; + + if (obj->has_global_gtt_mapping) { + vma->vm->clear_range(vma->vm, + vma->node.start, + obj->base.size, + true); + obj->has_global_gtt_mapping = 0; + } - obj->has_global_gtt_mapping = 0; + if (obj->has_aliasing_ppgtt_mapping) { + struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; + appgtt->base.clear_range(&appgtt->base, + vma->node.start, + obj->base.size, + true); + obj->has_aliasing_ppgtt_mapping = 0; + } } void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) @@ -1145,29 +1703,14 @@ void i915_gem_setup_global_gtt(struct drm_device *dev, /* Clear any non-preallocated blocks */ drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { - const unsigned long count = (hole_end - hole_start) / PAGE_SIZE; DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", hole_start, hole_end); - ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true); + ggtt_vm->clear_range(ggtt_vm, hole_start, + hole_end - hole_start, true); } /* And finally clear the reserved guard page */ - ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true); -} - -static bool -intel_enable_ppgtt(struct drm_device *dev) -{ - if (i915_enable_ppgtt >= 0) - return i915_enable_ppgtt; - -#ifdef CONFIG_INTEL_IOMMU - /* Disable ppgtt on SNB if VT-d is on. */ - if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) - return false; -#endif - - return true; + ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true); } void i915_gem_init_global_gtt(struct drm_device *dev) @@ -1178,26 +1721,6 @@ void i915_gem_init_global_gtt(struct drm_device *dev) gtt_size = dev_priv->gtt.base.total; mappable_size = dev_priv->gtt.mappable_end; - if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { - int ret; - - if (INTEL_INFO(dev)->gen <= 7) { - /* PPGTT pdes are stolen from global gtt ptes, so shrink the - * aperture accordingly when using aliasing ppgtt. */ - gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE; - } - - i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); - - ret = i915_gem_init_aliasing_ppgtt(dev); - if (!ret) - return; - - DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); - drm_mm_takedown(&dev_priv->gtt.base.mm); - if (INTEL_INFO(dev)->gen < 8) - gtt_size += GEN6_PPGTT_PD_ENTRIES*PAGE_SIZE; - } i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); } @@ -1252,14 +1775,27 @@ static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; if (bdw_gmch_ctl) bdw_gmch_ctl = 1 << bdw_gmch_ctl; - if (bdw_gmch_ctl > 4) { - WARN_ON(!i915_preliminary_hw_support); - return 4<<20; - } + +#ifdef CONFIG_X86_32 + /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */ + if (bdw_gmch_ctl > 4) + bdw_gmch_ctl = 4; +#endif return bdw_gmch_ctl << 20; } +static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) +{ + gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; + gmch_ctrl &= SNB_GMCH_GGMS_MASK; + + if (gmch_ctrl) + return 1 << (20 + gmch_ctrl); + + return 0; +} + static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) { snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; @@ -1274,6 +1810,24 @@ static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl) return bdw_gmch_ctl << 25; /* 32 MB units */ } +static size_t chv_get_stolen_size(u16 gmch_ctrl) +{ + gmch_ctrl >>= SNB_GMCH_GMS_SHIFT; + gmch_ctrl &= SNB_GMCH_GMS_MASK; + + /* + * 0x0 to 0x10: 32MB increments starting at 0MB + * 0x11 to 0x16: 4MB increments starting at 8MB + * 0x17 to 0x1d: 4MB increments start at 36MB + */ + if (gmch_ctrl < 0x11) + return gmch_ctrl << 25; + else if (gmch_ctrl < 0x17) + return (gmch_ctrl - 0x11 + 2) << 22; + else + return (gmch_ctrl - 0x17 + 9) << 22; +} + static int ggtt_probe_common(struct drm_device *dev, size_t gtt_size) { @@ -1304,19 +1858,8 @@ static int ggtt_probe_common(struct drm_device *dev, /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability * bits. When using advanced contexts each context stores its own PAT, but * writing this data shouldn't be harmful even in those cases. */ -static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv) -{ -#define GEN8_PPAT_UC (0<<0) -#define GEN8_PPAT_WC (1<<0) -#define GEN8_PPAT_WT (2<<0) -#define GEN8_PPAT_WB (3<<0) -#define GEN8_PPAT_ELLC_OVERRIDE (0<<2) -/* FIXME(BDW): Bspec is completely confused about cache control bits. */ -#define GEN8_PPAT_LLC (1<<2) -#define GEN8_PPAT_LLCELLC (2<<2) -#define GEN8_PPAT_LLCeLLC (3<<2) -#define GEN8_PPAT_AGE(x) (x<<4) -#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8)) +static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv) +{ uint64_t pat; pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */ @@ -1334,6 +1877,33 @@ static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv) I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); } +static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) +{ + uint64_t pat; + + /* + * Map WB on BDW to snooped on CHV. + * + * Only the snoop bit has meaning for CHV, the rest is + * ignored. + * + * Note that the harware enforces snooping for all page + * table accesses. The snoop bit is actually ignored for + * PDEs. + */ + pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) | + GEN8_PPAT(1, 0) | + GEN8_PPAT(2, 0) | + GEN8_PPAT(3, 0) | + GEN8_PPAT(4, CHV_PPAT_SNOOP) | + GEN8_PPAT(5, CHV_PPAT_SNOOP) | + GEN8_PPAT(6, CHV_PPAT_SNOOP) | + GEN8_PPAT(7, CHV_PPAT_SNOOP); + + I915_WRITE(GEN8_PRIVATE_PAT, pat); + I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); +} + static int gen8_gmch_probe(struct drm_device *dev, size_t *gtt_total, size_t *stolen, @@ -1354,12 +1924,20 @@ static int gen8_gmch_probe(struct drm_device *dev, pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); - *stolen = gen8_get_stolen_size(snb_gmch_ctl); + if (IS_CHERRYVIEW(dev)) { + *stolen = chv_get_stolen_size(snb_gmch_ctl); + gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); + } else { + *stolen = gen8_get_stolen_size(snb_gmch_ctl); + gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); + } - gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT; - gen8_setup_private_ppat(dev_priv); + if (IS_CHERRYVIEW(dev)) + chv_setup_private_ppat(dev_priv); + else + bdw_setup_private_ppat(dev_priv); ret = ggtt_probe_common(dev, gtt_size); @@ -1414,7 +1992,10 @@ static void gen6_gmch_remove(struct i915_address_space *vm) struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); - drm_mm_takedown(&vm->mm); + if (drm_mm_initialized(&vm->mm)) { + drm_mm_takedown(&vm->mm); + list_del(&vm->global_link); + } iounmap(gtt->gsm); teardown_scratch_page(vm->dev); } @@ -1438,7 +2019,6 @@ static int i915_gmch_probe(struct drm_device *dev, dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; - dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; if (unlikely(dev_priv->gtt.do_idle_maps)) DRM_INFO("applying Ironlake quirks for intel_iommu\n"); @@ -1448,6 +2028,10 @@ static int i915_gmch_probe(struct drm_device *dev, static void i915_gmch_remove(struct i915_address_space *vm) { + if (drm_mm_initialized(&vm->mm)) { + drm_mm_takedown(&vm->mm); + list_del(&vm->global_link); + } intel_gmch_remove(); } @@ -1490,6 +2074,77 @@ int i915_gem_gtt_init(struct drm_device *dev) gtt->base.total >> 20); DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20); DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); +#ifdef CONFIG_INTEL_IOMMU + if (intel_iommu_gfx_mapped) + DRM_INFO("VT-d active for gfx access\n"); +#endif + /* + * i915.enable_ppgtt is read-only, so do an early pass to validate the + * user's requested state against the hardware/driver capabilities. We + * do this now so that we can print out any log messages once rather + * than every time we check intel_enable_ppgtt(). + */ + i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); + DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); return 0; } + +static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj, + struct i915_address_space *vm) +{ + struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); + if (vma == NULL) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&vma->vma_link); + INIT_LIST_HEAD(&vma->mm_list); + INIT_LIST_HEAD(&vma->exec_list); + vma->vm = vm; + vma->obj = obj; + + switch (INTEL_INFO(vm->dev)->gen) { + case 8: + case 7: + case 6: + if (i915_is_ggtt(vm)) { + vma->unbind_vma = ggtt_unbind_vma; + vma->bind_vma = ggtt_bind_vma; + } else { + vma->unbind_vma = ppgtt_unbind_vma; + vma->bind_vma = ppgtt_bind_vma; + } + break; + case 5: + case 4: + case 3: + case 2: + BUG_ON(!i915_is_ggtt(vm)); + vma->unbind_vma = i915_ggtt_unbind_vma; + vma->bind_vma = i915_ggtt_bind_vma; + break; + default: + BUG(); + } + + /* Keep GGTT vmas first to make debug easier */ + if (i915_is_ggtt(vm)) + list_add(&vma->vma_link, &obj->vma_list); + else + list_add_tail(&vma->vma_link, &obj->vma_list); + + return vma; +} + +struct i915_vma * +i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, + struct i915_address_space *vm) +{ + struct i915_vma *vma; + + vma = i915_gem_obj_to_vma(obj, vm); + if (!vma) + vma = __i915_gem_vma_create(obj, vm); + + return vma; +} diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h new file mode 100644 index 00000000000..1b96a06be3c --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -0,0 +1,284 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Please try to maintain the following order within this file unless it makes + * sense to do otherwise. From top to bottom: + * 1. typedefs + * 2. #defines, and macros + * 3. structure definitions + * 4. function prototypes + * + * Within each section, please try to order by generation in ascending order, + * from top to bottom (ie. gen6 on the top, gen8 on the bottom). + */ + +#ifndef __I915_GEM_GTT_H__ +#define __I915_GEM_GTT_H__ + +typedef uint32_t gen6_gtt_pte_t; +typedef uint64_t gen8_gtt_pte_t; +typedef gen8_gtt_pte_t gen8_ppgtt_pde_t; + +#define gtt_total_entries(gtt) ((gtt).base.total >> PAGE_SHIFT) + +#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t)) +/* gen6-hsw has bit 11-4 for physical addr bit 39-32 */ +#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) +#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) +#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) +#define GEN6_PTE_CACHE_LLC (2 << 1) +#define GEN6_PTE_UNCACHED (1 << 1) +#define GEN6_PTE_VALID (1 << 0) + +#define GEN6_PPGTT_PD_ENTRIES 512 +#define GEN6_PD_SIZE (GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE) +#define GEN6_PD_ALIGN (PAGE_SIZE * 16) +#define GEN6_PDE_VALID (1 << 0) + +#define GEN7_PTE_CACHE_L3_LLC (3 << 1) + +#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) +#define BYT_PTE_WRITEABLE (1 << 1) + +/* Cacheability Control is a 4-bit value. The low three bits are stored in bits + * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. + */ +#define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ + (((bits) & 0x8) << (11 - 3))) +#define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) +#define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) +#define HSW_WB_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x8) +#define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) +#define HSW_WT_ELLC_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x7) +#define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) +#define HSW_PTE_UNCACHED (0) +#define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) +#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) + +/* GEN8 legacy style address is defined as a 3 level page table: + * 31:30 | 29:21 | 20:12 | 11:0 + * PDPE | PDE | PTE | offset + * The difference as compared to normal x86 3 level page table is the PDPEs are + * programmed via register. + */ +#define GEN8_PDPE_SHIFT 30 +#define GEN8_PDPE_MASK 0x3 +#define GEN8_PDE_SHIFT 21 +#define GEN8_PDE_MASK 0x1ff +#define GEN8_PTE_SHIFT 12 +#define GEN8_PTE_MASK 0x1ff +#define GEN8_LEGACY_PDPS 4 +#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t)) +#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t)) + +#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD) +#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */ +#define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */ +#define PPAT_DISPLAY_ELLC_INDEX _PAGE_PCD /* WT eLLC */ + +#define CHV_PPAT_SNOOP (1<<6) +#define GEN8_PPAT_AGE(x) (x<<4) +#define GEN8_PPAT_LLCeLLC (3<<2) +#define GEN8_PPAT_LLCELLC (2<<2) +#define GEN8_PPAT_LLC (1<<2) +#define GEN8_PPAT_WB (3<<0) +#define GEN8_PPAT_WT (2<<0) +#define GEN8_PPAT_WC (1<<0) +#define GEN8_PPAT_UC (0<<0) +#define GEN8_PPAT_ELLC_OVERRIDE (0<<2) +#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8)) + +enum i915_cache_level; +/** + * A VMA represents a GEM BO that is bound into an address space. Therefore, a + * VMA's presence cannot be guaranteed before binding, or after unbinding the + * object into/from the address space. + * + * To make things as simple as possible (ie. no refcounting), a VMA's lifetime + * will always be <= an objects lifetime. So object refcounting should cover us. + */ +struct i915_vma { + struct drm_mm_node node; + struct drm_i915_gem_object *obj; + struct i915_address_space *vm; + + /** This object's place on the active/inactive lists */ + struct list_head mm_list; + + struct list_head vma_link; /* Link in the object's VMA list */ + + /** This vma's place in the batchbuffer or on the eviction list */ + struct list_head exec_list; + + /** + * Used for performing relocations during execbuffer insertion. + */ + struct hlist_node exec_node; + unsigned long exec_handle; + struct drm_i915_gem_exec_object2 *exec_entry; + + /** + * How many users have pinned this object in GTT space. The following + * users can each hold at most one reference: pwrite/pread, pin_ioctl + * (via user_pin_count), execbuffer (objects are not allowed multiple + * times for the same batchbuffer), and the framebuffer code. When + * switching/pageflipping, the framebuffer code has at most two buffers + * pinned per crtc. + * + * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3 + * bits with absolutely no headroom. So use 4 bits. */ + unsigned int pin_count:4; +#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf + + /** Unmap an object from an address space. This usually consists of + * setting the valid PTE entries to a reserved scratch page. */ + void (*unbind_vma)(struct i915_vma *vma); + /* Map an object into an address space with the given cache flags. */ +#define GLOBAL_BIND (1<<0) + void (*bind_vma)(struct i915_vma *vma, + enum i915_cache_level cache_level, + u32 flags); +}; + +struct i915_address_space { + struct drm_mm mm; + struct drm_device *dev; + struct list_head global_link; + unsigned long start; /* Start offset always 0 for dri2 */ + size_t total; /* size addr space maps (ex. 2GB for ggtt) */ + + struct { + dma_addr_t addr; + struct page *page; + } scratch; + + /** + * List of objects currently involved in rendering. + * + * Includes buffers having the contents of their GPU caches + * flushed, not necessarily primitives. last_rendering_seqno + * represents when the rendering involved will be completed. + * + * A reference is held on the buffer while on this list. + */ + struct list_head active_list; + + /** + * LRU list of objects which are not in the ringbuffer and + * are ready to unbind, but are still in the GTT. + * + * last_rendering_seqno is 0 while an object is in this list. + * + * A reference is not held on the buffer while on this list, + * as merely being GTT-bound shouldn't prevent its being + * freed, and we'll pull it off the list in the free path. + */ + struct list_head inactive_list; + + /* FIXME: Need a more generic return type */ + gen6_gtt_pte_t (*pte_encode)(dma_addr_t addr, + enum i915_cache_level level, + bool valid); /* Create a valid PTE */ + void (*clear_range)(struct i915_address_space *vm, + uint64_t start, + uint64_t length, + bool use_scratch); + void (*insert_entries)(struct i915_address_space *vm, + struct sg_table *st, + uint64_t start, + enum i915_cache_level cache_level); + void (*cleanup)(struct i915_address_space *vm); +}; + +/* The Graphics Translation Table is the way in which GEN hardware translates a + * Graphics Virtual Address into a Physical Address. In addition to the normal + * collateral associated with any va->pa translations GEN hardware also has a + * portion of the GTT which can be mapped by the CPU and remain both coherent + * and correct (in cases like swizzling). That region is referred to as GMADR in + * the spec. + */ +struct i915_gtt { + struct i915_address_space base; + size_t stolen_size; /* Total size of stolen memory */ + + unsigned long mappable_end; /* End offset that we can CPU map */ + struct io_mapping *mappable; /* Mapping to our CPU mappable region */ + phys_addr_t mappable_base; /* PA of our GMADR */ + + /** "Graphics Stolen Memory" holds the global PTEs */ + void __iomem *gsm; + + bool do_idle_maps; + + int mtrr; + + /* global gtt ops */ + int (*gtt_probe)(struct drm_device *dev, size_t *gtt_total, + size_t *stolen, phys_addr_t *mappable_base, + unsigned long *mappable_end); +}; + +struct i915_hw_ppgtt { + struct i915_address_space base; + struct kref ref; + struct drm_mm_node node; + unsigned num_pd_entries; + unsigned num_pd_pages; /* gen8+ */ + union { + struct page **pt_pages; + struct page **gen8_pt_pages[GEN8_LEGACY_PDPS]; + }; + struct page *pd_pages; + union { + uint32_t pd_offset; + dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS]; + }; + union { + dma_addr_t *pt_dma_addr; + dma_addr_t *gen8_pt_dma_addr[4]; + }; + + struct intel_context *ctx; + + int (*enable)(struct i915_hw_ppgtt *ppgtt); + int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *ring, + bool synchronous); + void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); +}; + +int i915_gem_gtt_init(struct drm_device *dev); +void i915_gem_init_global_gtt(struct drm_device *dev); +void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start, + unsigned long mappable_end, unsigned long end); + +bool intel_enable_ppgtt(struct drm_device *dev, bool full); +int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt); + +void i915_check_and_clear_faults(struct drm_device *dev); +void i915_gem_suspend_gtt_mappings(struct drm_device *dev); +void i915_gem_restore_gtt_mappings(struct drm_device *dev); + +int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj); +void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); + +#endif diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c new file mode 100644 index 00000000000..34894b57306 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -0,0 +1,198 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Mika Kuoppala <mika.kuoppala@intel.com> + * + */ + +#include "i915_drv.h" +#include "intel_renderstate.h" + +struct i915_render_state { + struct drm_i915_gem_object *obj; + unsigned long ggtt_offset; + u32 *batch; + u32 size; + u32 len; +}; + +static struct i915_render_state *render_state_alloc(struct drm_device *dev) +{ + struct i915_render_state *so; + struct page *page; + int ret; + + so = kzalloc(sizeof(*so), GFP_KERNEL); + if (!so) + return ERR_PTR(-ENOMEM); + + so->obj = i915_gem_alloc_object(dev, 4096); + if (so->obj == NULL) { + ret = -ENOMEM; + goto free; + } + so->size = 4096; + + ret = i915_gem_obj_ggtt_pin(so->obj, 4096, 0); + if (ret) + goto free_gem; + + BUG_ON(so->obj->pages->nents != 1); + page = sg_page(so->obj->pages->sgl); + + so->batch = kmap(page); + if (!so->batch) { + ret = -ENOMEM; + goto unpin; + } + + so->ggtt_offset = i915_gem_obj_ggtt_offset(so->obj); + + return so; +unpin: + i915_gem_object_ggtt_unpin(so->obj); +free_gem: + drm_gem_object_unreference(&so->obj->base); +free: + kfree(so); + return ERR_PTR(ret); +} + +static void render_state_free(struct i915_render_state *so) +{ + kunmap(kmap_to_page(so->batch)); + i915_gem_object_ggtt_unpin(so->obj); + drm_gem_object_unreference(&so->obj->base); + kfree(so); +} + +static const struct intel_renderstate_rodata * +render_state_get_rodata(struct drm_device *dev, const int gen) +{ + switch (gen) { + case 6: + return &gen6_null_state; + case 7: + return &gen7_null_state; + case 8: + return &gen8_null_state; + } + + return NULL; +} + +static int render_state_setup(const int gen, + const struct intel_renderstate_rodata *rodata, + struct i915_render_state *so) +{ + const u64 goffset = i915_gem_obj_ggtt_offset(so->obj); + u32 reloc_index = 0; + u32 * const d = so->batch; + unsigned int i = 0; + int ret; + + if (!rodata || rodata->batch_items * 4 > so->size) + return -EINVAL; + + ret = i915_gem_object_set_to_cpu_domain(so->obj, true); + if (ret) + return ret; + + while (i < rodata->batch_items) { + u32 s = rodata->batch[i]; + + if (reloc_index < rodata->reloc_items && + i * 4 == rodata->reloc[reloc_index]) { + + s += goffset & 0xffffffff; + + /* We keep batch offsets max 32bit */ + if (gen >= 8) { + if (i + 1 >= rodata->batch_items || + rodata->batch[i + 1] != 0) + return -EINVAL; + + d[i] = s; + i++; + s = (goffset & 0xffffffff00000000ull) >> 32; + } + + reloc_index++; + } + + d[i] = s; + i++; + } + + ret = i915_gem_object_set_to_gtt_domain(so->obj, false); + if (ret) + return ret; + + if (rodata->reloc_items != reloc_index) { + DRM_ERROR("not all relocs resolved, %d out of %d\n", + reloc_index, rodata->reloc_items); + return -EINVAL; + } + + so->len = rodata->batch_items * 4; + + return 0; +} + +int i915_gem_render_state_init(struct intel_engine_cs *ring) +{ + const int gen = INTEL_INFO(ring->dev)->gen; + struct i915_render_state *so; + const struct intel_renderstate_rodata *rodata; + int ret; + + if (WARN_ON(ring->id != RCS)) + return -ENOENT; + + rodata = render_state_get_rodata(ring->dev, gen); + if (rodata == NULL) + return 0; + + so = render_state_alloc(ring->dev); + if (IS_ERR(so)) + return PTR_ERR(so); + + ret = render_state_setup(gen, rodata, so); + if (ret) + goto out; + + ret = ring->dispatch_execbuffer(ring, + i915_gem_obj_ggtt_offset(so->obj), + so->len, + I915_DISPATCH_SECURE); + if (ret) + goto out; + + i915_vma_move_to_active(i915_gem_obj_to_ggtt(so->obj), ring); + + ret = __i915_add_request(ring, NULL, so->obj, NULL); + /* __i915_add_request moves object to inactive if it fails */ +out: + render_state_free(so); + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 1a24e84f231..7465ab0fd39 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -74,6 +74,50 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) if (base == 0) return 0; + /* make sure we don't clobber the GTT if it's within stolen memory */ + if (INTEL_INFO(dev)->gen <= 4 && !IS_G33(dev) && !IS_G4X(dev)) { + struct { + u32 start, end; + } stolen[2] = { + { .start = base, .end = base + dev_priv->gtt.stolen_size, }, + { .start = base, .end = base + dev_priv->gtt.stolen_size, }, + }; + u64 gtt_start, gtt_end; + + gtt_start = I915_READ(PGTBL_CTL); + if (IS_GEN4(dev)) + gtt_start = (gtt_start & PGTBL_ADDRESS_LO_MASK) | + (gtt_start & PGTBL_ADDRESS_HI_MASK) << 28; + else + gtt_start &= PGTBL_ADDRESS_LO_MASK; + gtt_end = gtt_start + gtt_total_entries(dev_priv->gtt) * 4; + + if (gtt_start >= stolen[0].start && gtt_start < stolen[0].end) + stolen[0].end = gtt_start; + if (gtt_end > stolen[1].start && gtt_end <= stolen[1].end) + stolen[1].start = gtt_end; + + /* pick the larger of the two chunks */ + if (stolen[0].end - stolen[0].start > + stolen[1].end - stolen[1].start) { + base = stolen[0].start; + dev_priv->gtt.stolen_size = stolen[0].end - stolen[0].start; + } else { + base = stolen[1].start; + dev_priv->gtt.stolen_size = stolen[1].end - stolen[1].start; + } + + if (stolen[0].start != stolen[1].start || + stolen[0].end != stolen[1].end) { + DRM_DEBUG_KMS("GTT within stolen memory at 0x%llx-0x%llx\n", + (unsigned long long) gtt_start, + (unsigned long long) gtt_end - 1); + DRM_DEBUG_KMS("Stolen memory adjusted to 0x%x-0x%x\n", + base, base + (u32) dev_priv->gtt.stolen_size - 1); + } + } + + /* Verify that nothing else uses this physical address. Stolen * memory should be reserved by the BIOS and hidden from the * kernel. So if the region is already marked as busy, something @@ -82,9 +126,22 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) r = devm_request_mem_region(dev->dev, base, dev_priv->gtt.stolen_size, "Graphics Stolen Memory"); if (r == NULL) { - DRM_ERROR("conflict detected with stolen region: [0x%08x - 0x%08x]\n", - base, base + (uint32_t)dev_priv->gtt.stolen_size); - base = 0; + /* + * One more attempt but this time requesting region from + * base + 1, as we have seen that this resolves the region + * conflict with the PCI Bus. + * This is a BIOS w/a: Some BIOS wrap stolen in the root + * PCI bus, but have an off-by-one error. Hence retry the + * reservation starting from 1 instead of 0. + */ + r = devm_request_mem_region(dev->dev, base + 1, + dev_priv->gtt.stolen_size - 1, + "Graphics Stolen Memory"); + if (r == NULL) { + DRM_ERROR("conflict detected with stolen region: [0x%08x - 0x%08x]\n", + base, base + (uint32_t)dev_priv->gtt.stolen_size); + base = 0; + } } return base; @@ -201,6 +258,13 @@ int i915_gem_init_stolen(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int bios_reserved = 0; +#ifdef CONFIG_INTEL_IOMMU + if (intel_iommu_gfx_mapped && INTEL_INFO(dev)->gen < 8) { + DRM_INFO("DMAR active, disabling use of stolen memory\n"); + return 0; + } +#endif + if (dev_priv->gtt.stolen_size == 0) return 0; diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index b1390534804..cb150e8b433 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -87,7 +87,7 @@ void i915_gem_detect_bit_6_swizzle(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; @@ -294,7 +294,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_set_tiling *args = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; int ret = 0; @@ -308,7 +308,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, return -EINVAL; } - if (obj->pin_count || obj->framebuffer_references) { + if (i915_gem_obj_is_pinned(obj) || obj->framebuffer_references) { drm_gem_object_unreference_unlocked(&obj->base); return -EBUSY; } @@ -415,7 +415,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_get_tiling *args = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c new file mode 100644 index 00000000000..21ea92886a5 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -0,0 +1,711 @@ +/* + * Copyright © 2012-2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "drmP.h" +#include "i915_drm.h" +#include "i915_drv.h" +#include "i915_trace.h" +#include "intel_drv.h" +#include <linux/mmu_context.h> +#include <linux/mmu_notifier.h> +#include <linux/mempolicy.h> +#include <linux/swap.h> + +#if defined(CONFIG_MMU_NOTIFIER) +#include <linux/interval_tree.h> + +struct i915_mmu_notifier { + spinlock_t lock; + struct hlist_node node; + struct mmu_notifier mn; + struct rb_root objects; + struct drm_device *dev; + struct mm_struct *mm; + struct work_struct work; + unsigned long count; + unsigned long serial; +}; + +struct i915_mmu_object { + struct i915_mmu_notifier *mmu; + struct interval_tree_node it; + struct drm_i915_gem_object *obj; +}; + +static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn, + struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn); + struct interval_tree_node *it = NULL; + unsigned long serial = 0; + + end--; /* interval ranges are inclusive, but invalidate range is exclusive */ + while (start < end) { + struct drm_i915_gem_object *obj; + + obj = NULL; + spin_lock(&mn->lock); + if (serial == mn->serial) + it = interval_tree_iter_next(it, start, end); + else + it = interval_tree_iter_first(&mn->objects, start, end); + if (it != NULL) { + obj = container_of(it, struct i915_mmu_object, it)->obj; + drm_gem_object_reference(&obj->base); + serial = mn->serial; + } + spin_unlock(&mn->lock); + if (obj == NULL) + return; + + mutex_lock(&mn->dev->struct_mutex); + /* Cancel any active worker and force us to re-evaluate gup */ + obj->userptr.work = NULL; + + if (obj->pages != NULL) { + struct drm_i915_private *dev_priv = to_i915(mn->dev); + struct i915_vma *vma, *tmp; + bool was_interruptible; + + was_interruptible = dev_priv->mm.interruptible; + dev_priv->mm.interruptible = false; + + list_for_each_entry_safe(vma, tmp, &obj->vma_list, vma_link) { + int ret = i915_vma_unbind(vma); + WARN_ON(ret && ret != -EIO); + } + WARN_ON(i915_gem_object_put_pages(obj)); + + dev_priv->mm.interruptible = was_interruptible; + } + + start = obj->userptr.ptr + obj->base.size; + + drm_gem_object_unreference(&obj->base); + mutex_unlock(&mn->dev->struct_mutex); + } +} + +static const struct mmu_notifier_ops i915_gem_userptr_notifier = { + .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start, +}; + +static struct i915_mmu_notifier * +__i915_mmu_notifier_lookup(struct drm_device *dev, struct mm_struct *mm) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_mmu_notifier *mmu; + + /* Protected by dev->struct_mutex */ + hash_for_each_possible(dev_priv->mmu_notifiers, mmu, node, (unsigned long)mm) + if (mmu->mm == mm) + return mmu; + + return NULL; +} + +static struct i915_mmu_notifier * +i915_mmu_notifier_get(struct drm_device *dev, struct mm_struct *mm) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_mmu_notifier *mmu; + int ret; + + lockdep_assert_held(&dev->struct_mutex); + + mmu = __i915_mmu_notifier_lookup(dev, mm); + if (mmu) + return mmu; + + mmu = kmalloc(sizeof(*mmu), GFP_KERNEL); + if (mmu == NULL) + return ERR_PTR(-ENOMEM); + + spin_lock_init(&mmu->lock); + mmu->dev = dev; + mmu->mn.ops = &i915_gem_userptr_notifier; + mmu->mm = mm; + mmu->objects = RB_ROOT; + mmu->count = 0; + mmu->serial = 0; + + /* Protected by mmap_sem (write-lock) */ + ret = __mmu_notifier_register(&mmu->mn, mm); + if (ret) { + kfree(mmu); + return ERR_PTR(ret); + } + + /* Protected by dev->struct_mutex */ + hash_add(dev_priv->mmu_notifiers, &mmu->node, (unsigned long)mm); + return mmu; +} + +static void +__i915_mmu_notifier_destroy_worker(struct work_struct *work) +{ + struct i915_mmu_notifier *mmu = container_of(work, typeof(*mmu), work); + mmu_notifier_unregister(&mmu->mn, mmu->mm); + kfree(mmu); +} + +static void +__i915_mmu_notifier_destroy(struct i915_mmu_notifier *mmu) +{ + lockdep_assert_held(&mmu->dev->struct_mutex); + + /* Protected by dev->struct_mutex */ + hash_del(&mmu->node); + + /* Our lock ordering is: mmap_sem, mmu_notifier_scru, struct_mutex. + * We enter the function holding struct_mutex, therefore we need + * to drop our mutex prior to calling mmu_notifier_unregister in + * order to prevent lock inversion (and system-wide deadlock) + * between the mmap_sem and struct-mutex. Hence we defer the + * unregistration to a workqueue where we hold no locks. + */ + INIT_WORK(&mmu->work, __i915_mmu_notifier_destroy_worker); + schedule_work(&mmu->work); +} + +static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mmu) +{ + if (++mmu->serial == 0) + mmu->serial = 1; +} + +static void +i915_mmu_notifier_del(struct i915_mmu_notifier *mmu, + struct i915_mmu_object *mn) +{ + lockdep_assert_held(&mmu->dev->struct_mutex); + + spin_lock(&mmu->lock); + interval_tree_remove(&mn->it, &mmu->objects); + __i915_mmu_notifier_update_serial(mmu); + spin_unlock(&mmu->lock); + + /* Protected against _add() by dev->struct_mutex */ + if (--mmu->count == 0) + __i915_mmu_notifier_destroy(mmu); +} + +static int +i915_mmu_notifier_add(struct i915_mmu_notifier *mmu, + struct i915_mmu_object *mn) +{ + struct interval_tree_node *it; + int ret; + + ret = i915_mutex_lock_interruptible(mmu->dev); + if (ret) + return ret; + + /* Make sure we drop the final active reference (and thereby + * remove the objects from the interval tree) before we do + * the check for overlapping objects. + */ + i915_gem_retire_requests(mmu->dev); + + /* Disallow overlapping userptr objects */ + spin_lock(&mmu->lock); + it = interval_tree_iter_first(&mmu->objects, + mn->it.start, mn->it.last); + if (it) { + struct drm_i915_gem_object *obj; + + /* We only need to check the first object in the range as it + * either has cancelled gup work queued and we need to + * return back to the user to give time for the gup-workers + * to flush their object references upon which the object will + * be removed from the interval-tree, or the the range is + * still in use by another client and the overlap is invalid. + */ + + obj = container_of(it, struct i915_mmu_object, it)->obj; + ret = obj->userptr.workers ? -EAGAIN : -EINVAL; + } else { + interval_tree_insert(&mn->it, &mmu->objects); + __i915_mmu_notifier_update_serial(mmu); + ret = 0; + } + spin_unlock(&mmu->lock); + mutex_unlock(&mmu->dev->struct_mutex); + + return ret; +} + +static void +i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) +{ + struct i915_mmu_object *mn; + + mn = obj->userptr.mn; + if (mn == NULL) + return; + + i915_mmu_notifier_del(mn->mmu, mn); + obj->userptr.mn = NULL; +} + +static int +i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, + unsigned flags) +{ + struct i915_mmu_notifier *mmu; + struct i915_mmu_object *mn; + int ret; + + if (flags & I915_USERPTR_UNSYNCHRONIZED) + return capable(CAP_SYS_ADMIN) ? 0 : -EPERM; + + down_write(&obj->userptr.mm->mmap_sem); + ret = i915_mutex_lock_interruptible(obj->base.dev); + if (ret == 0) { + mmu = i915_mmu_notifier_get(obj->base.dev, obj->userptr.mm); + if (!IS_ERR(mmu)) + mmu->count++; /* preemptive add to act as a refcount */ + else + ret = PTR_ERR(mmu); + mutex_unlock(&obj->base.dev->struct_mutex); + } + up_write(&obj->userptr.mm->mmap_sem); + if (ret) + return ret; + + mn = kzalloc(sizeof(*mn), GFP_KERNEL); + if (mn == NULL) { + ret = -ENOMEM; + goto destroy_mmu; + } + + mn->mmu = mmu; + mn->it.start = obj->userptr.ptr; + mn->it.last = mn->it.start + obj->base.size - 1; + mn->obj = obj; + + ret = i915_mmu_notifier_add(mmu, mn); + if (ret) + goto free_mn; + + obj->userptr.mn = mn; + return 0; + +free_mn: + kfree(mn); +destroy_mmu: + mutex_lock(&obj->base.dev->struct_mutex); + if (--mmu->count == 0) + __i915_mmu_notifier_destroy(mmu); + mutex_unlock(&obj->base.dev->struct_mutex); + return ret; +} + +#else + +static void +i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj) +{ +} + +static int +i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj, + unsigned flags) +{ + if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0) + return -ENODEV; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + return 0; +} +#endif + +struct get_pages_work { + struct work_struct work; + struct drm_i915_gem_object *obj; + struct task_struct *task; +}; + + +#if IS_ENABLED(CONFIG_SWIOTLB) +#define swiotlb_active() swiotlb_nr_tbl() +#else +#define swiotlb_active() 0 +#endif + +static int +st_set_pages(struct sg_table **st, struct page **pvec, int num_pages) +{ + struct scatterlist *sg; + int ret, n; + + *st = kmalloc(sizeof(**st), GFP_KERNEL); + if (*st == NULL) + return -ENOMEM; + + if (swiotlb_active()) { + ret = sg_alloc_table(*st, num_pages, GFP_KERNEL); + if (ret) + goto err; + + for_each_sg((*st)->sgl, sg, num_pages, n) + sg_set_page(sg, pvec[n], PAGE_SIZE, 0); + } else { + ret = sg_alloc_table_from_pages(*st, pvec, num_pages, + 0, num_pages << PAGE_SHIFT, + GFP_KERNEL); + if (ret) + goto err; + } + + return 0; + +err: + kfree(*st); + *st = NULL; + return ret; +} + +static void +__i915_gem_userptr_get_pages_worker(struct work_struct *_work) +{ + struct get_pages_work *work = container_of(_work, typeof(*work), work); + struct drm_i915_gem_object *obj = work->obj; + struct drm_device *dev = obj->base.dev; + const int num_pages = obj->base.size >> PAGE_SHIFT; + struct page **pvec; + int pinned, ret; + + ret = -ENOMEM; + pinned = 0; + + pvec = kmalloc(num_pages*sizeof(struct page *), + GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); + if (pvec == NULL) + pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); + if (pvec != NULL) { + struct mm_struct *mm = obj->userptr.mm; + + down_read(&mm->mmap_sem); + while (pinned < num_pages) { + ret = get_user_pages(work->task, mm, + obj->userptr.ptr + pinned * PAGE_SIZE, + num_pages - pinned, + !obj->userptr.read_only, 0, + pvec + pinned, NULL); + if (ret < 0) + break; + + pinned += ret; + } + up_read(&mm->mmap_sem); + } + + mutex_lock(&dev->struct_mutex); + if (obj->userptr.work != &work->work) { + ret = 0; + } else if (pinned == num_pages) { + ret = st_set_pages(&obj->pages, pvec, num_pages); + if (ret == 0) { + list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list); + pinned = 0; + } + } + + obj->userptr.work = ERR_PTR(ret); + obj->userptr.workers--; + drm_gem_object_unreference(&obj->base); + mutex_unlock(&dev->struct_mutex); + + release_pages(pvec, pinned, 0); + drm_free_large(pvec); + + put_task_struct(work->task); + kfree(work); +} + +static int +i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) +{ + const int num_pages = obj->base.size >> PAGE_SHIFT; + struct page **pvec; + int pinned, ret; + + /* If userspace should engineer that these pages are replaced in + * the vma between us binding this page into the GTT and completion + * of rendering... Their loss. If they change the mapping of their + * pages they need to create a new bo to point to the new vma. + * + * However, that still leaves open the possibility of the vma + * being copied upon fork. Which falls under the same userspace + * synchronisation issue as a regular bo, except that this time + * the process may not be expecting that a particular piece of + * memory is tied to the GPU. + * + * Fortunately, we can hook into the mmu_notifier in order to + * discard the page references prior to anything nasty happening + * to the vma (discard or cloning) which should prevent the more + * egregious cases from causing harm. + */ + + pvec = NULL; + pinned = 0; + if (obj->userptr.mm == current->mm) { + pvec = kmalloc(num_pages*sizeof(struct page *), + GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY); + if (pvec == NULL) { + pvec = drm_malloc_ab(num_pages, sizeof(struct page *)); + if (pvec == NULL) + return -ENOMEM; + } + + pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, + !obj->userptr.read_only, pvec); + } + if (pinned < num_pages) { + if (pinned < 0) { + ret = pinned; + pinned = 0; + } else { + /* Spawn a worker so that we can acquire the + * user pages without holding our mutex. Access + * to the user pages requires mmap_sem, and we have + * a strict lock ordering of mmap_sem, struct_mutex - + * we already hold struct_mutex here and so cannot + * call gup without encountering a lock inversion. + * + * Userspace will keep on repeating the operation + * (thanks to EAGAIN) until either we hit the fast + * path or the worker completes. If the worker is + * cancelled or superseded, the task is still run + * but the results ignored. (This leads to + * complications that we may have a stray object + * refcount that we need to be wary of when + * checking for existing objects during creation.) + * If the worker encounters an error, it reports + * that error back to this function through + * obj->userptr.work = ERR_PTR. + */ + ret = -EAGAIN; + if (obj->userptr.work == NULL && + obj->userptr.workers < I915_GEM_USERPTR_MAX_WORKERS) { + struct get_pages_work *work; + + work = kmalloc(sizeof(*work), GFP_KERNEL); + if (work != NULL) { + obj->userptr.work = &work->work; + obj->userptr.workers++; + + work->obj = obj; + drm_gem_object_reference(&obj->base); + + work->task = current; + get_task_struct(work->task); + + INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker); + schedule_work(&work->work); + } else + ret = -ENOMEM; + } else { + if (IS_ERR(obj->userptr.work)) { + ret = PTR_ERR(obj->userptr.work); + obj->userptr.work = NULL; + } + } + } + } else { + ret = st_set_pages(&obj->pages, pvec, num_pages); + if (ret == 0) { + obj->userptr.work = NULL; + pinned = 0; + } + } + + release_pages(pvec, pinned, 0); + drm_free_large(pvec); + return ret; +} + +static void +i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) +{ + struct scatterlist *sg; + int i; + + BUG_ON(obj->userptr.work != NULL); + + if (obj->madv != I915_MADV_WILLNEED) + obj->dirty = 0; + + for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { + struct page *page = sg_page(sg); + + if (obj->dirty) + set_page_dirty(page); + + mark_page_accessed(page); + page_cache_release(page); + } + obj->dirty = 0; + + sg_free_table(obj->pages); + kfree(obj->pages); +} + +static void +i915_gem_userptr_release(struct drm_i915_gem_object *obj) +{ + i915_gem_userptr_release__mmu_notifier(obj); + + if (obj->userptr.mm) { + mmput(obj->userptr.mm); + obj->userptr.mm = NULL; + } +} + +static int +i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) +{ + if (obj->userptr.mn) + return 0; + + return i915_gem_userptr_init__mmu_notifier(obj, 0); +} + +static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { + .dmabuf_export = i915_gem_userptr_dmabuf_export, + .get_pages = i915_gem_userptr_get_pages, + .put_pages = i915_gem_userptr_put_pages, + .release = i915_gem_userptr_release, +}; + +/** + * Creates a new mm object that wraps some normal memory from the process + * context - user memory. + * + * We impose several restrictions upon the memory being mapped + * into the GPU. + * 1. It must be page aligned (both start/end addresses, i.e ptr and size). + * 2. It cannot overlap any other userptr object in the same address space. + * 3. It must be normal system memory, not a pointer into another map of IO + * space (e.g. it must not be a GTT mmapping of another object). + * 4. We only allow a bo as large as we could in theory map into the GTT, + * that is we limit the size to the total size of the GTT. + * 5. The bo is marked as being snoopable. The backing pages are left + * accessible directly by the CPU, but reads and writes by the GPU may + * incur the cost of a snoop (unless you have an LLC architecture). + * + * Synchronisation between multiple users and the GPU is left to userspace + * through the normal set-domain-ioctl. The kernel will enforce that the + * GPU relinquishes the VMA before it is returned back to the system + * i.e. upon free(), munmap() or process termination. However, the userspace + * malloc() library may not immediately relinquish the VMA after free() and + * instead reuse it whilst the GPU is still reading and writing to the VMA. + * Caveat emptor. + * + * Also note, that the object created here is not currently a "first class" + * object, in that several ioctls are banned. These are the CPU access + * ioctls: mmap(), pwrite and pread. In practice, you are expected to use + * direct access via your pointer rather than use those ioctls. + * + * If you think this is a good interface to use to pass GPU memory between + * drivers, please use dma-buf instead. In fact, wherever possible use + * dma-buf instead. + */ +int +i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_userptr *args = data; + struct drm_i915_gem_object *obj; + int ret; + u32 handle; + + if (args->flags & ~(I915_USERPTR_READ_ONLY | + I915_USERPTR_UNSYNCHRONIZED)) + return -EINVAL; + + if (offset_in_page(args->user_ptr | args->user_size)) + return -EINVAL; + + if (args->user_size > dev_priv->gtt.base.total) + return -E2BIG; + + if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE, + (char __user *)(unsigned long)args->user_ptr, args->user_size)) + return -EFAULT; + + if (args->flags & I915_USERPTR_READ_ONLY) { + /* On almost all of the current hw, we cannot tell the GPU that a + * page is readonly, so this is just a placeholder in the uAPI. + */ + return -ENODEV; + } + + /* Allocate the new object */ + obj = i915_gem_object_alloc(dev); + if (obj == NULL) + return -ENOMEM; + + drm_gem_private_object_init(dev, &obj->base, args->user_size); + i915_gem_object_init(obj, &i915_gem_userptr_ops); + obj->cache_level = I915_CACHE_LLC; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + + obj->userptr.ptr = args->user_ptr; + obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY); + + /* And keep a pointer to the current->mm for resolving the user pages + * at binding. This means that we need to hook into the mmu_notifier + * in order to detect if the mmu is destroyed. + */ + ret = -ENOMEM; + if ((obj->userptr.mm = get_task_mm(current))) + ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); + if (ret == 0) + ret = drm_gem_handle_create(file, &obj->base, &handle); + + /* drop reference from allocate - handle holds it now */ + drm_gem_object_unreference_unlocked(&obj->base); + if (ret) + return ret; + + args->handle = handle; + return 0; +} + +int +i915_gem_init_userptr(struct drm_device *dev) +{ +#if defined(CONFIG_MMU_NOTIFIER) + struct drm_i915_private *dev_priv = to_i915(dev); + hash_init(dev_priv->mmu_notifiers); +#endif + return 0; +} diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index d7fd2fd2f0a..66cf41765bf 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -42,6 +42,7 @@ static const char *ring_str(int ring) case VCS: return "bsd"; case BCS: return "blt"; case VECS: return "vebox"; + case VCS2: return "bsd2"; default: return ""; } } @@ -146,7 +147,10 @@ static void i915_error_vprintf(struct drm_i915_error_state_buf *e, va_list tmp; va_copy(tmp, args); - if (!__i915_error_seek(e, vsnprintf(NULL, 0, f, tmp))) + len = vsnprintf(NULL, 0, f, tmp); + va_end(tmp); + + if (!__i915_error_seek(e, len)) return; } @@ -201,6 +205,7 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m, err_puts(m, tiling_flag(err->tiling)); err_puts(m, dirty_flag(err->dirty)); err_puts(m, purgeable_flag(err->purgeable)); + err_puts(m, err->userptr ? " userptr" : ""); err_puts(m, err->ring != -1 ? " " : ""); err_puts(m, ring_str(err->ring)); err_puts(m, i915_cache_level_str(err->cache_level)); @@ -235,50 +240,62 @@ static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a) static void i915_ring_error_state(struct drm_i915_error_state_buf *m, struct drm_device *dev, - struct drm_i915_error_state *error, - unsigned ring) + struct drm_i915_error_ring *ring) { - BUG_ON(ring >= I915_NUM_RINGS); /* shut up confused gcc */ - if (!error->ring[ring].valid) + if (!ring->valid) return; - err_printf(m, "%s command stream:\n", ring_str(ring)); - err_printf(m, " HEAD: 0x%08x\n", error->head[ring]); - err_printf(m, " TAIL: 0x%08x\n", error->tail[ring]); - err_printf(m, " CTL: 0x%08x\n", error->ctl[ring]); - err_printf(m, " ACTHD: 0x%08x\n", error->acthd[ring]); - err_printf(m, " IPEIR: 0x%08x\n", error->ipeir[ring]); - err_printf(m, " IPEHR: 0x%08x\n", error->ipehr[ring]); - err_printf(m, " INSTDONE: 0x%08x\n", error->instdone[ring]); + err_printf(m, " HEAD: 0x%08x\n", ring->head); + err_printf(m, " TAIL: 0x%08x\n", ring->tail); + err_printf(m, " CTL: 0x%08x\n", ring->ctl); + err_printf(m, " HWS: 0x%08x\n", ring->hws); + err_printf(m, " ACTHD: 0x%08x %08x\n", (u32)(ring->acthd>>32), (u32)ring->acthd); + err_printf(m, " IPEIR: 0x%08x\n", ring->ipeir); + err_printf(m, " IPEHR: 0x%08x\n", ring->ipehr); + err_printf(m, " INSTDONE: 0x%08x\n", ring->instdone); if (INTEL_INFO(dev)->gen >= 4) { - err_printf(m, " BBADDR: 0x%08llx\n", error->bbaddr[ring]); - err_printf(m, " BB_STATE: 0x%08x\n", error->bbstate[ring]); - err_printf(m, " INSTPS: 0x%08x\n", error->instps[ring]); + err_printf(m, " BBADDR: 0x%08x %08x\n", (u32)(ring->bbaddr>>32), (u32)ring->bbaddr); + err_printf(m, " BB_STATE: 0x%08x\n", ring->bbstate); + err_printf(m, " INSTPS: 0x%08x\n", ring->instps); } - err_printf(m, " INSTPM: 0x%08x\n", error->instpm[ring]); - err_printf(m, " FADDR: 0x%08x\n", error->faddr[ring]); + err_printf(m, " INSTPM: 0x%08x\n", ring->instpm); + err_printf(m, " FADDR: 0x%08x %08x\n", upper_32_bits(ring->faddr), + lower_32_bits(ring->faddr)); if (INTEL_INFO(dev)->gen >= 6) { - err_printf(m, " RC PSMI: 0x%08x\n", error->rc_psmi[ring]); - err_printf(m, " FAULT_REG: 0x%08x\n", error->fault_reg[ring]); + err_printf(m, " RC PSMI: 0x%08x\n", ring->rc_psmi); + err_printf(m, " FAULT_REG: 0x%08x\n", ring->fault_reg); err_printf(m, " SYNC_0: 0x%08x [last synced 0x%08x]\n", - error->semaphore_mboxes[ring][0], - error->semaphore_seqno[ring][0]); + ring->semaphore_mboxes[0], + ring->semaphore_seqno[0]); err_printf(m, " SYNC_1: 0x%08x [last synced 0x%08x]\n", - error->semaphore_mboxes[ring][1], - error->semaphore_seqno[ring][1]); + ring->semaphore_mboxes[1], + ring->semaphore_seqno[1]); if (HAS_VEBOX(dev)) { err_printf(m, " SYNC_2: 0x%08x [last synced 0x%08x]\n", - error->semaphore_mboxes[ring][2], - error->semaphore_seqno[ring][2]); + ring->semaphore_mboxes[2], + ring->semaphore_seqno[2]); + } + } + if (USES_PPGTT(dev)) { + err_printf(m, " GFX_MODE: 0x%08x\n", ring->vm_info.gfx_mode); + + if (INTEL_INFO(dev)->gen >= 8) { + int i; + for (i = 0; i < 4; i++) + err_printf(m, " PDP%d: 0x%016llx\n", + i, ring->vm_info.pdp[i]); + } else { + err_printf(m, " PP_DIR_BASE: 0x%08x\n", + ring->vm_info.pp_dir_base); } } - err_printf(m, " seqno: 0x%08x\n", error->seqno[ring]); - err_printf(m, " waiting: %s\n", yesno(error->waiting[ring])); - err_printf(m, " ring->head: 0x%08x\n", error->cpu_ring_head[ring]); - err_printf(m, " ring->tail: 0x%08x\n", error->cpu_ring_tail[ring]); + err_printf(m, " seqno: 0x%08x\n", ring->seqno); + err_printf(m, " waiting: %s\n", yesno(ring->waiting)); + err_printf(m, " ring->head: 0x%08x\n", ring->cpu_ring_head); + err_printf(m, " ring->tail: 0x%08x\n", ring->cpu_ring_tail); err_printf(m, " hangcheck: %s [%d]\n", - hangcheck_action_to_str(error->hangcheck_action[ring]), - error->hangcheck_score[ring]); + hangcheck_action_to_str(ring->hangcheck_action), + ring->hangcheck_score); } void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...) @@ -290,22 +307,54 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...) va_end(args); } +static void print_error_obj(struct drm_i915_error_state_buf *m, + struct drm_i915_error_object *obj) +{ + int page, offset, elt; + + for (page = offset = 0; page < obj->page_count; page++) { + for (elt = 0; elt < PAGE_SIZE/4; elt++) { + err_printf(m, "%08x : %08x\n", offset, + obj->pages[page][elt]); + offset += 4; + } + } +} + int i915_error_state_to_str(struct drm_i915_error_state_buf *m, const struct i915_error_state_file_priv *error_priv) { struct drm_device *dev = error_priv->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_error_state *error = error_priv->error; - int i, j, page, offset, elt; + int i, j, offset, elt; + int max_hangcheck_score; if (!error) { err_printf(m, "no error state collected\n"); goto out; } + err_printf(m, "%s\n", error->error_msg); err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec, error->time.tv_usec); err_printf(m, "Kernel: " UTS_RELEASE "\n"); + max_hangcheck_score = 0; + for (i = 0; i < ARRAY_SIZE(error->ring); i++) { + if (error->ring[i].hangcheck_score > max_hangcheck_score) + max_hangcheck_score = error->ring[i].hangcheck_score; + } + for (i = 0; i < ARRAY_SIZE(error->ring); i++) { + if (error->ring[i].hangcheck_score == max_hangcheck_score && + error->ring[i].pid != -1) { + err_printf(m, "Active process (on ring %s): %s [%d]\n", + ring_str(i), + error->ring[i].comm, + error->ring[i].pid); + } + } + err_printf(m, "Reset count: %u\n", error->reset_count); + err_printf(m, "Suspend count: %u\n", error->suspend_count); err_printf(m, "PCI ID: 0x%04x\n", dev->pdev->device); err_printf(m, "EIR: 0x%08x\n", error->eir); err_printf(m, "IER: 0x%08x\n", error->ier); @@ -330,8 +379,10 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, if (INTEL_INFO(dev)->gen == 7) err_printf(m, "ERR_INT: 0x%08x\n", error->err_int); - for (i = 0; i < ARRAY_SIZE(error->ring); i++) - i915_ring_error_state(m, dev, error, i); + for (i = 0; i < ARRAY_SIZE(error->ring); i++) { + err_printf(m, "%s command stream:\n", ring_str(i)); + i915_ring_error_state(m, dev, &error->ring[i]); + } if (error->active_bo) print_error_buffers(m, "Active", @@ -346,18 +397,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, for (i = 0; i < ARRAY_SIZE(error->ring); i++) { struct drm_i915_error_object *obj; - if ((obj = error->ring[i].batchbuffer)) { - err_printf(m, "%s --- gtt_offset = 0x%08x\n", - dev_priv->ring[i].name, + obj = error->ring[i].batchbuffer; + if (obj) { + err_puts(m, dev_priv->ring[i].name); + if (error->ring[i].pid != -1) + err_printf(m, " (submitted by %s [%d])", + error->ring[i].comm, + error->ring[i].pid); + err_printf(m, " --- gtt_offset = 0x%08x\n", obj->gtt_offset); - offset = 0; - for (page = 0; page < obj->page_count; page++) { - for (elt = 0; elt < PAGE_SIZE/4; elt++) { - err_printf(m, "%08x : %08x\n", offset, - obj->pages[page][elt]); - offset += 4; - } - } + print_error_obj(m, obj); + } + + obj = error->ring[i].wa_batchbuffer; + if (obj) { + err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n", + dev_priv->ring[i].name, obj->gtt_offset); + print_error_obj(m, obj); } if (error->ring[i].num_requests) { @@ -376,19 +432,11 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, "%s --- ringbuffer = 0x%08x\n", dev_priv->ring[i].name, obj->gtt_offset); - offset = 0; - for (page = 0; page < obj->page_count; page++) { - for (elt = 0; elt < PAGE_SIZE/4; elt++) { - err_printf(m, "%08x : %08x\n", - offset, - obj->pages[page][elt]); - offset += 4; - } - } + print_error_obj(m, obj); } - if ((obj = error->ring[i].ctx)) { - err_printf(m, "%s --- HW Context = 0x%08x\n", + if ((obj = error->ring[i].hws_page)) { + err_printf(m, "%s --- HW Status = 0x%08x\n", dev_priv->ring[i].name, obj->gtt_offset); offset = 0; @@ -402,6 +450,13 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, offset += 16; } } + + if ((obj = error->ring[i].ctx)) { + err_printf(m, "%s --- HW Context = 0x%08x\n", + dev_priv->ring[i].name, + obj->gtt_offset); + print_error_obj(m, obj); + } } if (error->overlay) @@ -469,6 +524,7 @@ static void i915_error_state_free(struct kref *error_ref) for (i = 0; i < ARRAY_SIZE(error->ring); i++) { i915_error_object_free(error->ring[i].batchbuffer); i915_error_object_free(error->ring[i].ringbuffer); + i915_error_object_free(error->ring[i].hws_page); i915_error_object_free(error->ring[i].ctx); kfree(error->ring[i].requests); } @@ -482,6 +538,7 @@ static void i915_error_state_free(struct kref *error_ref) static struct drm_i915_error_object * i915_error_object_create_sized(struct drm_i915_private *dev_priv, struct drm_i915_gem_object *src, + struct i915_address_space *vm, const int num_pages) { struct drm_i915_error_object *dst; @@ -495,7 +552,7 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv, if (dst == NULL) return NULL; - reloc_offset = dst->gtt_offset = i915_gem_obj_ggtt_offset(src); + reloc_offset = dst->gtt_offset = i915_gem_obj_offset(src, vm); for (i = 0; i < num_pages; i++) { unsigned long flags; void *d; @@ -505,8 +562,10 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv, goto unwind; local_irq_save(flags); - if (reloc_offset < dev_priv->gtt.mappable_end && - src->has_global_gtt_mapping) { + if (src->cache_level == I915_CACHE_NONE && + reloc_offset < dev_priv->gtt.mappable_end && + src->has_global_gtt_mapping && + i915_is_ggtt(vm)) { void __iomem *s; /* Simply ignore tiling or any overlapping fence. @@ -556,8 +615,12 @@ unwind: kfree(dst); return NULL; } -#define i915_error_object_create(dev_priv, src) \ - i915_error_object_create_sized((dev_priv), (src), \ +#define i915_error_object_create(dev_priv, src, vm) \ + i915_error_object_create_sized((dev_priv), (src), (vm), \ + (src)->base.size>>PAGE_SHIFT) + +#define i915_error_ggtt_object_create(dev_priv, src) \ + i915_error_object_create_sized((dev_priv), (src), &(dev_priv)->gtt.base, \ (src)->base.size>>PAGE_SHIFT) static void capture_bo(struct drm_i915_error_buffer *err, @@ -572,13 +635,14 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->write_domain = obj->base.write_domain; err->fence_reg = obj->fence_reg; err->pinned = 0; - if (obj->pin_count > 0) + if (i915_gem_obj_is_pinned(obj)) err->pinned = 1; if (obj->user_pin_count > 0) err->pinned = -1; err->tiling = obj->tiling_mode; err->dirty = obj->dirty; err->purgeable = obj->madv != I915_MADV_WILLNEED; + err->userptr = obj->userptr.mm != NULL; err->ring = obj->ring ? obj->ring->id : -1; err->cache_level = obj->cache_level; } @@ -605,7 +669,7 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err, int i = 0; list_for_each_entry(obj, head, global_list) { - if (obj->pin_count == 0) + if (!i915_gem_obj_is_pinned(obj)) continue; capture_bo(err++, obj); @@ -616,6 +680,39 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err, return i; } +/* Generate a semi-unique error code. The code is not meant to have meaning, The + * code's only purpose is to try to prevent false duplicated bug reports by + * grossly estimating a GPU error state. + * + * TODO Ideally, hashing the batchbuffer would be a very nice way to determine + * the hang if we could strip the GTT offset information from it. + * + * It's only a small step better than a random number in its current form. + */ +static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv, + struct drm_i915_error_state *error, + int *ring_id) +{ + uint32_t error_code = 0; + int i; + + /* IPEHR would be an ideal way to detect errors, as it's the gross + * measure of "the command that hung." However, has some very common + * synchronization commands which almost always appear in the case + * strictly a client bug. Use instdone to differentiate those some. + */ + for (i = 0; i < I915_NUM_RINGS; i++) { + if (error->ring[i].hangcheck_action == HANGCHECK_HUNG) { + if (ring_id) + *ring_id = i; + + return error->ring[i].ipehr ^ error->ring[i].instdone; + } + } + + return error_code; +} + static void i915_gem_record_fences(struct drm_device *dev, struct drm_i915_error_state *error) { @@ -649,111 +746,120 @@ static void i915_gem_record_fences(struct drm_device *dev, } } -static struct drm_i915_error_object * -i915_error_first_batchbuffer(struct drm_i915_private *dev_priv, - struct intel_ring_buffer *ring) -{ - struct i915_address_space *vm; - struct i915_vma *vma; - struct drm_i915_gem_object *obj; - u32 seqno; - - if (!ring->get_seqno) - return NULL; - - if (HAS_BROKEN_CS_TLB(dev_priv->dev)) { - u32 acthd = I915_READ(ACTHD); - - if (WARN_ON(ring->id != RCS)) - return NULL; - - obj = ring->scratch.obj; - if (obj != NULL && - acthd >= i915_gem_obj_ggtt_offset(obj) && - acthd < i915_gem_obj_ggtt_offset(obj) + obj->base.size) - return i915_error_object_create(dev_priv, obj); - } - - seqno = ring->get_seqno(ring, false); - list_for_each_entry(vm, &dev_priv->vm_list, global_link) { - list_for_each_entry(vma, &vm->active_list, mm_list) { - obj = vma->obj; - if (obj->ring != ring) - continue; - - if (i915_seqno_passed(seqno, obj->last_read_seqno)) - continue; - - if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0) - continue; - - /* We need to copy these to an anonymous buffer as the simplest - * method to avoid being overwritten by userspace. - */ - return i915_error_object_create(dev_priv, obj); - } - } - - return NULL; -} - static void i915_record_ring_state(struct drm_device *dev, - struct drm_i915_error_state *error, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring, + struct drm_i915_error_ring *ering) { struct drm_i915_private *dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen >= 6) { - error->rc_psmi[ring->id] = I915_READ(ring->mmio_base + 0x50); - error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring)); - error->semaphore_mboxes[ring->id][0] + ering->rc_psmi = I915_READ(ring->mmio_base + 0x50); + ering->fault_reg = I915_READ(RING_FAULT_REG(ring)); + ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(ring->mmio_base)); - error->semaphore_mboxes[ring->id][1] + ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(ring->mmio_base)); - error->semaphore_seqno[ring->id][0] = ring->sync_seqno[0]; - error->semaphore_seqno[ring->id][1] = ring->sync_seqno[1]; + ering->semaphore_seqno[0] = ring->semaphore.sync_seqno[0]; + ering->semaphore_seqno[1] = ring->semaphore.sync_seqno[1]; } if (HAS_VEBOX(dev)) { - error->semaphore_mboxes[ring->id][2] = + ering->semaphore_mboxes[2] = I915_READ(RING_SYNC_2(ring->mmio_base)); - error->semaphore_seqno[ring->id][2] = ring->sync_seqno[2]; + ering->semaphore_seqno[2] = ring->semaphore.sync_seqno[2]; } if (INTEL_INFO(dev)->gen >= 4) { - error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base)); - error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base)); - error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base)); - error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base)); - error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base)); - error->bbaddr[ring->id] = I915_READ(RING_BBADDR(ring->mmio_base)); - if (INTEL_INFO(dev)->gen >= 8) - error->bbaddr[ring->id] |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32; - error->bbstate[ring->id] = I915_READ(RING_BBSTATE(ring->mmio_base)); + ering->faddr = I915_READ(RING_DMA_FADD(ring->mmio_base)); + ering->ipeir = I915_READ(RING_IPEIR(ring->mmio_base)); + ering->ipehr = I915_READ(RING_IPEHR(ring->mmio_base)); + ering->instdone = I915_READ(RING_INSTDONE(ring->mmio_base)); + ering->instps = I915_READ(RING_INSTPS(ring->mmio_base)); + ering->bbaddr = I915_READ(RING_BBADDR(ring->mmio_base)); + if (INTEL_INFO(dev)->gen >= 8) { + ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(ring->mmio_base)) << 32; + ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32; + } + ering->bbstate = I915_READ(RING_BBSTATE(ring->mmio_base)); } else { - error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX); - error->ipeir[ring->id] = I915_READ(IPEIR); - error->ipehr[ring->id] = I915_READ(IPEHR); - error->instdone[ring->id] = I915_READ(INSTDONE); + ering->faddr = I915_READ(DMA_FADD_I8XX); + ering->ipeir = I915_READ(IPEIR); + ering->ipehr = I915_READ(IPEHR); + ering->instdone = I915_READ(INSTDONE); + } + + ering->waiting = waitqueue_active(&ring->irq_queue); + ering->instpm = I915_READ(RING_INSTPM(ring->mmio_base)); + ering->seqno = ring->get_seqno(ring, false); + ering->acthd = intel_ring_get_active_head(ring); + ering->head = I915_READ_HEAD(ring); + ering->tail = I915_READ_TAIL(ring); + ering->ctl = I915_READ_CTL(ring); + + if (I915_NEED_GFX_HWS(dev)) { + int mmio; + + if (IS_GEN7(dev)) { + switch (ring->id) { + default: + case RCS: + mmio = RENDER_HWS_PGA_GEN7; + break; + case BCS: + mmio = BLT_HWS_PGA_GEN7; + break; + case VCS: + mmio = BSD_HWS_PGA_GEN7; + break; + case VECS: + mmio = VEBOX_HWS_PGA_GEN7; + break; + } + } else if (IS_GEN6(ring->dev)) { + mmio = RING_HWS_PGA_GEN6(ring->mmio_base); + } else { + /* XXX: gen8 returns to sanity */ + mmio = RING_HWS_PGA(ring->mmio_base); + } + + ering->hws = I915_READ(mmio); } - error->waiting[ring->id] = waitqueue_active(&ring->irq_queue); - error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base)); - error->seqno[ring->id] = ring->get_seqno(ring, false); - error->acthd[ring->id] = intel_ring_get_active_head(ring); - error->head[ring->id] = I915_READ_HEAD(ring); - error->tail[ring->id] = I915_READ_TAIL(ring); - error->ctl[ring->id] = I915_READ_CTL(ring); + ering->cpu_ring_head = ring->buffer->head; + ering->cpu_ring_tail = ring->buffer->tail; - error->cpu_ring_head[ring->id] = ring->head; - error->cpu_ring_tail[ring->id] = ring->tail; + ering->hangcheck_score = ring->hangcheck.score; + ering->hangcheck_action = ring->hangcheck.action; - error->hangcheck_score[ring->id] = ring->hangcheck.score; - error->hangcheck_action[ring->id] = ring->hangcheck.action; + if (USES_PPGTT(dev)) { + int i; + + ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(ring)); + + switch (INTEL_INFO(dev)->gen) { + case 8: + for (i = 0; i < 4; i++) { + ering->vm_info.pdp[i] = + I915_READ(GEN8_RING_PDP_UDW(ring, i)); + ering->vm_info.pdp[i] <<= 32; + ering->vm_info.pdp[i] |= + I915_READ(GEN8_RING_PDP_LDW(ring, i)); + } + break; + case 7: + ering->vm_info.pp_dir_base = + I915_READ(RING_PP_DIR_BASE(ring)); + break; + case 6: + ering->vm_info.pp_dir_base = + I915_READ(RING_PP_DIR_BASE_READ(ring)); + break; + } + } } -static void i915_gem_record_active_context(struct intel_ring_buffer *ring, +static void i915_gem_record_active_context(struct intel_engine_cs *ring, struct drm_i915_error_state *error, struct drm_i915_error_ring *ering) { @@ -766,8 +872,7 @@ static void i915_gem_record_active_context(struct intel_ring_buffer *ring, list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { if ((error->ccid & PAGE_MASK) == i915_gem_obj_ggtt_offset(obj)) { - ering->ctx = i915_error_object_create_sized(dev_priv, - obj, 1); + ering->ctx = i915_error_ggtt_object_create(dev_priv, obj); break; } } @@ -781,21 +886,56 @@ static void i915_gem_record_rings(struct drm_device *dev, int i, count; for (i = 0; i < I915_NUM_RINGS; i++) { - struct intel_ring_buffer *ring = &dev_priv->ring[i]; + struct intel_engine_cs *ring = &dev_priv->ring[i]; + + error->ring[i].pid = -1; if (ring->dev == NULL) continue; error->ring[i].valid = true; - i915_record_ring_state(dev, error, ring); + i915_record_ring_state(dev, ring, &error->ring[i]); - error->ring[i].batchbuffer = - i915_error_first_batchbuffer(dev_priv, ring); + request = i915_gem_find_active_request(ring); + if (request) { + /* We need to copy these to an anonymous buffer + * as the simplest method to avoid being overwritten + * by userspace. + */ + error->ring[i].batchbuffer = + i915_error_object_create(dev_priv, + request->batch_obj, + request->ctx ? + request->ctx->vm : + &dev_priv->gtt.base); + + if (HAS_BROKEN_CS_TLB(dev_priv->dev) && + ring->scratch.obj) + error->ring[i].wa_batchbuffer = + i915_error_ggtt_object_create(dev_priv, + ring->scratch.obj); + + if (request->file_priv) { + struct task_struct *task; + + rcu_read_lock(); + task = pid_task(request->file_priv->file->pid, + PIDTYPE_PID); + if (task) { + strcpy(error->ring[i].comm, task->comm); + error->ring[i].pid = task->pid; + } + rcu_read_unlock(); + } + } error->ring[i].ringbuffer = - i915_error_object_create(dev_priv, ring->obj); + i915_error_ggtt_object_create(dev_priv, ring->buffer->obj); + if (ring->status_page.obj) + error->ring[i].hws_page = + i915_error_ggtt_object_create(dev_priv, ring->status_page.obj); i915_gem_record_active_context(ring, error, &error->ring[i]); @@ -842,7 +982,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv, i++; error->active_bo_count[ndx] = i; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) - if (obj->pin_count) + if (i915_gem_obj_is_pinned(obj)) i++; error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx]; @@ -876,11 +1016,6 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv, list_for_each_entry(vm, &dev_priv->vm_list, global_link) cnt++; - if (WARN(cnt > 1, "Multiple VMs not yet supported\n")) - cnt = 1; - - vm = &dev_priv->gtt.base; - error->active_bo = kcalloc(cnt, sizeof(*error->active_bo), GFP_ATOMIC); error->pinned_bo = kcalloc(cnt, sizeof(*error->pinned_bo), GFP_ATOMIC); error->active_bo_count = kcalloc(cnt, sizeof(*error->active_bo_count), @@ -892,6 +1027,105 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv, i915_gem_capture_vm(dev_priv, error, vm, i++); } +/* Capture all registers which don't fit into another category. */ +static void i915_capture_reg_state(struct drm_i915_private *dev_priv, + struct drm_i915_error_state *error) +{ + struct drm_device *dev = dev_priv->dev; + + /* General organization + * 1. Registers specific to a single generation + * 2. Registers which belong to multiple generations + * 3. Feature specific registers. + * 4. Everything else + * Please try to follow the order. + */ + + /* 1: Registers specific to a single generation */ + if (IS_VALLEYVIEW(dev)) { + error->ier = I915_READ(GTIER) | I915_READ(VLV_IER); + error->forcewake = I915_READ(FORCEWAKE_VLV); + } + + if (IS_GEN7(dev)) + error->err_int = I915_READ(GEN7_ERR_INT); + + if (IS_GEN6(dev)) { + error->forcewake = I915_READ(FORCEWAKE); + error->gab_ctl = I915_READ(GAB_CTL); + error->gfx_mode = I915_READ(GFX_MODE); + } + + /* 2: Registers which belong to multiple generations */ + if (INTEL_INFO(dev)->gen >= 7) + error->forcewake = I915_READ(FORCEWAKE_MT); + + if (INTEL_INFO(dev)->gen >= 6) { + error->derrmr = I915_READ(DERRMR); + error->error = I915_READ(ERROR_GEN6); + error->done_reg = I915_READ(DONE_REG); + } + + /* 3: Feature specific registers */ + if (IS_GEN6(dev) || IS_GEN7(dev)) { + error->gam_ecochk = I915_READ(GAM_ECOCHK); + error->gac_eco = I915_READ(GAC_ECO_BITS); + } + + /* 4: Everything else */ + if (HAS_HW_CONTEXTS(dev)) + error->ccid = I915_READ(CCID); + + if (HAS_PCH_SPLIT(dev)) + error->ier = I915_READ(DEIER) | I915_READ(GTIER); + else { + if (IS_GEN2(dev)) + error->ier = I915_READ16(IER); + else + error->ier = I915_READ(IER); + } + + /* 4: Everything else */ + error->eir = I915_READ(EIR); + error->pgtbl_er = I915_READ(PGTBL_ER); + + i915_get_extra_instdone(dev, error->extra_instdone); +} + +static void i915_error_capture_msg(struct drm_device *dev, + struct drm_i915_error_state *error, + bool wedged, + const char *error_msg) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 ecode; + int ring_id = -1, len; + + ecode = i915_error_generate_code(dev_priv, error, &ring_id); + + len = scnprintf(error->error_msg, sizeof(error->error_msg), + "GPU HANG: ecode %d:0x%08x", ring_id, ecode); + + if (ring_id != -1 && error->ring[ring_id].pid != -1) + len += scnprintf(error->error_msg + len, + sizeof(error->error_msg) - len, + ", in %s [%d]", + error->ring[ring_id].comm, + error->ring[ring_id].pid); + + scnprintf(error->error_msg + len, sizeof(error->error_msg) - len, + ", reason: %s, action: %s", + error_msg, + wedged ? "reset" : "continue"); +} + +static void i915_capture_gen_state(struct drm_i915_private *dev_priv, + struct drm_i915_error_state *error) +{ + error->reset_count = i915_reset_count(&dev_priv->gpu_error); + error->suspend_count = dev_priv->suspend_count; +} + /** * i915_capture_error_state - capture an error record for later analysis * @dev: drm device @@ -901,18 +1135,13 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv, * out a structure which becomes available in debugfs for user level tools * to pick up. */ -void i915_capture_error_state(struct drm_device *dev) +void i915_capture_error_state(struct drm_device *dev, bool wedged, + const char *error_msg) { + static bool warned; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_error_state *error; unsigned long flags; - int pipe; - - spin_lock_irqsave(&dev_priv->gpu_error.lock, flags); - error = dev_priv->gpu_error.first_error; - spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags); - if (error) - return; /* Account for pipe specific data like PIPE*STAT */ error = kzalloc(sizeof(*error), GFP_ATOMIC); @@ -921,52 +1150,10 @@ void i915_capture_error_state(struct drm_device *dev) return; } - DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", - dev->primary->index); - DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n"); - DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n"); - DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n"); - DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n"); - kref_init(&error->ref); - error->eir = I915_READ(EIR); - error->pgtbl_er = I915_READ(PGTBL_ER); - if (HAS_HW_CONTEXTS(dev)) - error->ccid = I915_READ(CCID); - - if (HAS_PCH_SPLIT(dev)) - error->ier = I915_READ(DEIER) | I915_READ(GTIER); - else if (IS_VALLEYVIEW(dev)) - error->ier = I915_READ(GTIER) | I915_READ(VLV_IER); - else if (IS_GEN2(dev)) - error->ier = I915_READ16(IER); - else - error->ier = I915_READ(IER); - - if (INTEL_INFO(dev)->gen >= 6) - error->derrmr = I915_READ(DERRMR); - - if (IS_VALLEYVIEW(dev)) - error->forcewake = I915_READ(FORCEWAKE_VLV); - else if (INTEL_INFO(dev)->gen >= 7) - error->forcewake = I915_READ(FORCEWAKE_MT); - else if (INTEL_INFO(dev)->gen == 6) - error->forcewake = I915_READ(FORCEWAKE); - - if (!HAS_PCH_SPLIT(dev)) - for_each_pipe(pipe) - error->pipestat[pipe] = I915_READ(PIPESTAT(pipe)); - - if (INTEL_INFO(dev)->gen >= 6) { - error->error = I915_READ(ERROR_GEN6); - error->done_reg = I915_READ(DONE_REG); - } - - if (INTEL_INFO(dev)->gen == 7) - error->err_int = I915_READ(GEN7_ERR_INT); - - i915_get_extra_instdone(dev, error->extra_instdone); + i915_capture_gen_state(dev_priv, error); + i915_capture_reg_state(dev_priv, error); i915_gem_capture_buffers(dev_priv, error); i915_gem_record_fences(dev, error); i915_gem_record_rings(dev, error); @@ -976,6 +1163,9 @@ void i915_capture_error_state(struct drm_device *dev) error->overlay = intel_overlay_capture_error_state(dev); error->display = intel_display_capture_error_state(dev); + i915_error_capture_msg(dev, error, wedged, error_msg); + DRM_INFO("%s\n", error->error_msg); + spin_lock_irqsave(&dev_priv->gpu_error.lock, flags); if (dev_priv->gpu_error.first_error == NULL) { dev_priv->gpu_error.first_error = error; @@ -983,8 +1173,19 @@ void i915_capture_error_state(struct drm_device *dev) } spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags); - if (error) + if (error) { i915_error_state_free(&error->ref); + return; + } + + if (!warned) { + DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n"); + DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n"); + DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n"); + DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n"); + DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", dev->primary->index); + warned = true; + } } void i915_error_state_get(struct drm_device *dev, diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index 3c59584161c..2e0613e2625 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -208,7 +208,7 @@ long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (nr < DRM_COMMAND_BASE) return drm_compat_ioctl(filp, cmd, arg); - if (nr < DRM_COMMAND_BASE + DRM_ARRAY_SIZE(i915_compat_ioctls)) + if (nr < DRM_COMMAND_BASE + ARRAY_SIZE(i915_compat_ioctls)) fn = i915_compat_ioctls[nr - DRM_COMMAND_BASE]; if (fn != NULL) diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 17d8fcb1b6f..c05c84f3f09 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -80,17 +80,64 @@ static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */ [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS }; +/* IIR can theoretically queue up two events. Be paranoid. */ +#define GEN8_IRQ_RESET_NDX(type, which) do { \ + I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \ + POSTING_READ(GEN8_##type##_IMR(which)); \ + I915_WRITE(GEN8_##type##_IER(which), 0); \ + I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ + POSTING_READ(GEN8_##type##_IIR(which)); \ + I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ + POSTING_READ(GEN8_##type##_IIR(which)); \ +} while (0) + +#define GEN5_IRQ_RESET(type) do { \ + I915_WRITE(type##IMR, 0xffffffff); \ + POSTING_READ(type##IMR); \ + I915_WRITE(type##IER, 0); \ + I915_WRITE(type##IIR, 0xffffffff); \ + POSTING_READ(type##IIR); \ + I915_WRITE(type##IIR, 0xffffffff); \ + POSTING_READ(type##IIR); \ +} while (0) + +/* + * We should clear IMR at preinstall/uninstall, and just check at postinstall. + */ +#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \ + u32 val = I915_READ(reg); \ + if (val) { \ + WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \ + (reg), val); \ + I915_WRITE((reg), 0xffffffff); \ + POSTING_READ(reg); \ + I915_WRITE((reg), 0xffffffff); \ + POSTING_READ(reg); \ + } \ +} while (0) + +#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \ + GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \ + I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \ + I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \ + POSTING_READ(GEN8_##type##_IER(which)); \ +} while (0) + +#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \ + GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \ + I915_WRITE(type##IMR, (imr_val)); \ + I915_WRITE(type##IER, (ier_val)); \ + POSTING_READ(type##IER); \ +} while (0) + /* For display hotplug interrupt */ static void -ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) +ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask) { assert_spin_locked(&dev_priv->irq_lock); - if (dev_priv->pc8.irqs_disabled) { - WARN(1, "IRQs disabled\n"); - dev_priv->pc8.regsave.deimr &= ~mask; + if (WARN_ON(dev_priv->pm.irqs_disabled)) return; - } if ((dev_priv->irq_mask & mask) != 0) { dev_priv->irq_mask &= ~mask; @@ -100,15 +147,12 @@ ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) } static void -ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask) +ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask) { assert_spin_locked(&dev_priv->irq_lock); - if (dev_priv->pc8.irqs_disabled) { - WARN(1, "IRQs disabled\n"); - dev_priv->pc8.regsave.deimr |= mask; + if (WARN_ON(dev_priv->pm.irqs_disabled)) return; - } if ((dev_priv->irq_mask & mask) != mask) { dev_priv->irq_mask |= mask; @@ -129,13 +173,8 @@ static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, { assert_spin_locked(&dev_priv->irq_lock); - if (dev_priv->pc8.irqs_disabled) { - WARN(1, "IRQs disabled\n"); - dev_priv->pc8.regsave.gtimr &= ~interrupt_mask; - dev_priv->pc8.regsave.gtimr |= (~enabled_irq_mask & - interrupt_mask); + if (WARN_ON(dev_priv->pm.irqs_disabled)) return; - } dev_priv->gt_irq_mask &= ~interrupt_mask; dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask); @@ -167,13 +206,8 @@ static void snb_update_pm_irq(struct drm_i915_private *dev_priv, assert_spin_locked(&dev_priv->irq_lock); - if (dev_priv->pc8.irqs_disabled) { - WARN(1, "IRQs disabled\n"); - dev_priv->pc8.regsave.gen6_pmimr &= ~interrupt_mask; - dev_priv->pc8.regsave.gen6_pmimr |= (~enabled_irq_mask & - interrupt_mask); + if (WARN_ON(dev_priv->pm.irqs_disabled)) return; - } new_val = dev_priv->pm_irq_mask; new_val &= ~interrupt_mask; @@ -214,6 +248,46 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) return true; } +/** + * bdw_update_pm_irq - update GT interrupt 2 + * @dev_priv: driver private + * @interrupt_mask: mask of interrupt bits to update + * @enabled_irq_mask: mask of interrupt bits to enable + * + * Copied from the snb function, updated with relevant register offsets + */ +static void bdw_update_pm_irq(struct drm_i915_private *dev_priv, + uint32_t interrupt_mask, + uint32_t enabled_irq_mask) +{ + uint32_t new_val; + + assert_spin_locked(&dev_priv->irq_lock); + + if (WARN_ON(dev_priv->pm.irqs_disabled)) + return; + + new_val = dev_priv->pm_irq_mask; + new_val &= ~interrupt_mask; + new_val |= (~enabled_irq_mask & interrupt_mask); + + if (new_val != dev_priv->pm_irq_mask) { + dev_priv->pm_irq_mask = new_val; + I915_WRITE(GEN8_GT_IMR(2), dev_priv->pm_irq_mask); + POSTING_READ(GEN8_GT_IMR(2)); + } +} + +void bdw_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) +{ + bdw_update_pm_irq(dev_priv, mask, mask); +} + +void bdw_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) +{ + bdw_update_pm_irq(dev_priv, mask, 0); +} + static bool cpt_can_enable_serr_int(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -232,6 +306,53 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) return true; } +void i9xx_check_fifo_underruns(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *crtc; + unsigned long flags; + + spin_lock_irqsave(&dev_priv->irq_lock, flags); + + for_each_intel_crtc(dev, crtc) { + u32 reg = PIPESTAT(crtc->pipe); + u32 pipestat; + + if (crtc->cpu_fifo_underrun_disabled) + continue; + + pipestat = I915_READ(reg) & 0xffff0000; + if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0) + continue; + + I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS); + POSTING_READ(reg); + + DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe)); + } + + spin_unlock_irqrestore(&dev_priv->irq_lock, flags); +} + +static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, + enum pipe pipe, + bool enable, bool old) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 reg = PIPESTAT(pipe); + u32 pipestat = I915_READ(reg) & 0xffff0000; + + assert_spin_locked(&dev_priv->irq_lock); + + if (enable) { + I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS); + POSTING_READ(reg); + } else { + if (old && pipestat & PIPE_FIFO_UNDERRUN_STATUS) + DRM_ERROR("pipe %c underrun\n", pipe_name(pipe)); + } +} + static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { @@ -246,7 +367,8 @@ static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev, } static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev, - enum pipe pipe, bool enable) + enum pipe pipe, + bool enable, bool old) { struct drm_i915_private *dev_priv = dev->dev_private; if (enable) { @@ -257,15 +379,12 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev, ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB); } else { - bool was_enabled = !(I915_READ(DEIMR) & DE_ERR_INT_IVB); - - /* Change the state _after_ we've read out the current one. */ ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB); - if (!was_enabled && - (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe))) { - DRM_DEBUG_KMS("uncleared fifo underrun on pipe %c\n", - pipe_name(pipe)); + if (old && + I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe)) { + DRM_ERROR("uncleared fifo underrun on pipe %c\n", + pipe_name(pipe)); } } } @@ -301,14 +420,8 @@ static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv, assert_spin_locked(&dev_priv->irq_lock); - if (dev_priv->pc8.irqs_disabled && - (interrupt_mask & SDE_HOTPLUG_MASK_CPT)) { - WARN(1, "IRQs disabled\n"); - dev_priv->pc8.regsave.sdeimr &= ~interrupt_mask; - dev_priv->pc8.regsave.sdeimr |= (~enabled_irq_mask & - interrupt_mask); + if (WARN_ON(dev_priv->pm.irqs_disabled)) return; - } I915_WRITE(SDEIMR, sdeimr); POSTING_READ(SDEIMR); @@ -334,7 +447,7 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev, static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, enum transcoder pch_transcoder, - bool enable) + bool enable, bool old) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -347,16 +460,12 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT); } else { - uint32_t tmp = I915_READ(SERR_INT); - bool was_enabled = !(I915_READ(SDEIMR) & SDE_ERROR_CPT); - - /* Change the state _after_ we've read out the current one. */ ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT); - if (!was_enabled && - (tmp & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder))) { - DRM_DEBUG_KMS("uncleared pch fifo underrun on pch transcoder %c\n", - transcoder_name(pch_transcoder)); + if (old && I915_READ(SERR_INT) & + SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) { + DRM_ERROR("uncleared pch fifo underrun on pch transcoder %c\n", + transcoder_name(pch_transcoder)); } } } @@ -375,36 +484,55 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, * * Returns the previous state of underrun reporting. */ -bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, - enum pipe pipe, bool enable) +static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, + enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - unsigned long flags; - bool ret; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - - ret = !intel_crtc->cpu_fifo_underrun_disabled; + bool old; - if (enable == ret) - goto done; + assert_spin_locked(&dev_priv->irq_lock); + old = !intel_crtc->cpu_fifo_underrun_disabled; intel_crtc->cpu_fifo_underrun_disabled = !enable; - if (IS_GEN5(dev) || IS_GEN6(dev)) + if (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev)) + i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old); + else if (IS_GEN5(dev) || IS_GEN6(dev)) ironlake_set_fifo_underrun_reporting(dev, pipe, enable); else if (IS_GEN7(dev)) - ivybridge_set_fifo_underrun_reporting(dev, pipe, enable); + ivybridge_set_fifo_underrun_reporting(dev, pipe, enable, old); else if (IS_GEN8(dev)) broadwell_set_fifo_underrun_reporting(dev, pipe, enable); -done: + return old; +} + +bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, + enum pipe pipe, bool enable) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long flags; + bool ret; + + spin_lock_irqsave(&dev_priv->irq_lock, flags); + ret = __intel_set_cpu_fifo_underrun_reporting(dev, pipe, enable); spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + return ret; } +static bool __cpu_fifo_underrun_reporting_enabled(struct drm_device *dev, + enum pipe pipe) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + + return !intel_crtc->cpu_fifo_underrun_disabled; +} + /** * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages * @dev: drm device @@ -427,7 +555,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev, struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); unsigned long flags; - bool ret; + bool old; /* * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT @@ -440,63 +568,132 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev, spin_lock_irqsave(&dev_priv->irq_lock, flags); - ret = !intel_crtc->pch_fifo_underrun_disabled; - - if (enable == ret) - goto done; - + old = !intel_crtc->pch_fifo_underrun_disabled; intel_crtc->pch_fifo_underrun_disabled = !enable; if (HAS_PCH_IBX(dev)) ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable); else - cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable); + cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable, old); -done: spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - return ret; + return old; } -void -i915_enable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask) +static void +__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, + u32 enable_mask, u32 status_mask) { u32 reg = PIPESTAT(pipe); - u32 pipestat = I915_READ(reg) & 0x7fff0000; + u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; assert_spin_locked(&dev_priv->irq_lock); - if ((pipestat & mask) == mask) + if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || + status_mask & ~PIPESTAT_INT_STATUS_MASK, + "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", + pipe_name(pipe), enable_mask, status_mask)) + return; + + if ((pipestat & enable_mask) == enable_mask) return; + dev_priv->pipestat_irq_mask[pipe] |= status_mask; + /* Enable the interrupt, clear any pending status */ - pipestat |= mask | (mask >> 16); + pipestat |= enable_mask | status_mask; I915_WRITE(reg, pipestat); POSTING_READ(reg); } -void -i915_disable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask) +static void +__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, + u32 enable_mask, u32 status_mask) { u32 reg = PIPESTAT(pipe); - u32 pipestat = I915_READ(reg) & 0x7fff0000; + u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK; assert_spin_locked(&dev_priv->irq_lock); - if ((pipestat & mask) == 0) + if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK || + status_mask & ~PIPESTAT_INT_STATUS_MASK, + "pipe %c: enable_mask=0x%x, status_mask=0x%x\n", + pipe_name(pipe), enable_mask, status_mask)) return; - pipestat &= ~mask; + if ((pipestat & enable_mask) == 0) + return; + + dev_priv->pipestat_irq_mask[pipe] &= ~status_mask; + + pipestat &= ~enable_mask; I915_WRITE(reg, pipestat); POSTING_READ(reg); } +static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask) +{ + u32 enable_mask = status_mask << 16; + + /* + * On pipe A we don't support the PSR interrupt yet, + * on pipe B and C the same bit MBZ. + */ + if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV)) + return 0; + /* + * On pipe B and C we don't support the PSR interrupt yet, on pipe + * A the same bit is for perf counters which we don't use either. + */ + if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV)) + return 0; + + enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS | + SPRITE0_FLIP_DONE_INT_EN_VLV | + SPRITE1_FLIP_DONE_INT_EN_VLV); + if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV) + enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV; + if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV) + enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV; + + return enable_mask; +} + +void +i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, + u32 status_mask) +{ + u32 enable_mask; + + if (IS_VALLEYVIEW(dev_priv->dev)) + enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev, + status_mask); + else + enable_mask = status_mask << 16; + __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask); +} + +void +i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, + u32 status_mask) +{ + u32 enable_mask; + + if (IS_VALLEYVIEW(dev_priv->dev)) + enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev, + status_mask); + else + enable_mask = status_mask << 16; + __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask); +} + /** * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion */ static void i915_enable_asle_pipestat(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; if (!dev_priv->opregion.asle || !IS_MOBILE(dev)) @@ -504,10 +701,10 @@ static void i915_enable_asle_pipestat(struct drm_device *dev) spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_ENABLE); + i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS); if (INTEL_INFO(dev)->gen >= 4) i915_enable_pipestat(dev_priv, PIPE_A, - PIPE_LEGACY_BLC_EVENT_ENABLE); + PIPE_LEGACY_BLC_EVENT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } @@ -524,7 +721,7 @@ static void i915_enable_asle_pipestat(struct drm_device *dev) static int i915_pipe_enabled(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; if (drm_core_check_feature(dev, DRIVER_MODESET)) { /* Locking is horribly broken here, but whatever. */ @@ -537,6 +734,56 @@ i915_pipe_enabled(struct drm_device *dev, int pipe) } } +/* + * This timing diagram depicts the video signal in and + * around the vertical blanking period. + * + * Assumptions about the fictitious mode used in this example: + * vblank_start >= 3 + * vsync_start = vblank_start + 1 + * vsync_end = vblank_start + 2 + * vtotal = vblank_start + 3 + * + * start of vblank: + * latch double buffered registers + * increment frame counter (ctg+) + * generate start of vblank interrupt (gen4+) + * | + * | frame start: + * | generate frame start interrupt (aka. vblank interrupt) (gmch) + * | may be shifted forward 1-3 extra lines via PIPECONF + * | | + * | | start of vsync: + * | | generate vsync interrupt + * | | | + * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx + * . \hs/ . \hs/ \hs/ \hs/ . \hs/ + * ----va---> <-----------------vb--------------------> <--------va------------- + * | | <----vs-----> | + * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2) + * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+) + * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi) + * | | | + * last visible pixel first visible pixel + * | increment frame counter (gen3/4) + * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4) + * + * x = horizontal active + * _ = horizontal blanking + * hs = horizontal sync + * va = vertical active + * vb = vertical blanking + * vs = vertical sync + * vbs = vblank_start (number) + * + * Summary: + * - most events happen at the start of horizontal sync + * - frame start happens at the start of horizontal blank, 1-4 lines + * (depending on PIPECONF settings) after the start of vblank + * - gen3/4 pixel and frame counter are synchronized with the start + * of horizontal active on the first line of vertical active + */ + static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe) { /* Gen2 doesn't have a hardware frame counter */ @@ -548,10 +795,10 @@ static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe) */ static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long high_frame; unsigned long low_frame; - u32 high1, high2, low, pixel, vbl_start; + u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; if (!i915_pipe_enabled(dev, pipe)) { DRM_DEBUG_DRIVER("trying to get vblank count for disabled " @@ -565,18 +812,28 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode; - vbl_start = mode->crtc_vblank_start * mode->crtc_htotal; + htotal = mode->crtc_htotal; + hsync_start = mode->crtc_hsync_start; + vbl_start = mode->crtc_vblank_start; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + vbl_start = DIV_ROUND_UP(vbl_start, 2); } else { - enum transcoder cpu_transcoder = - intel_pipe_to_cpu_transcoder(dev_priv, pipe); - u32 htotal; + enum transcoder cpu_transcoder = (enum transcoder) pipe; htotal = ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff) + 1; + hsync_start = (I915_READ(HSYNC(cpu_transcoder)) & 0x1fff) + 1; vbl_start = (I915_READ(VBLANK(cpu_transcoder)) & 0x1fff) + 1; - - vbl_start *= htotal; + if ((I915_READ(PIPECONF(cpu_transcoder)) & + PIPECONF_INTERLACE_MASK) != PIPECONF_PROGRESSIVE) + vbl_start = DIV_ROUND_UP(vbl_start, 2); } + /* Convert to pixel count */ + vbl_start *= htotal; + + /* Start of vblank event occurs at start of hsync */ + vbl_start -= htotal - hsync_start; + high_frame = PIPEFRAME(pipe); low_frame = PIPEFRAMEPIXEL(pipe); @@ -605,7 +862,7 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int reg = PIPE_FRMCOUNT_GM45(pipe); if (!i915_pipe_enabled(dev, pipe)) { @@ -619,33 +876,29 @@ static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe) /* raw reads, only for fast reads of display block, no need for forcewake etc. */ #define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__)) -#define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__)) -static bool ilk_pipe_in_vblank_locked(struct drm_device *dev, enum pipe pipe) +static int __intel_get_crtc_scanline(struct intel_crtc *crtc) { + struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t status; + const struct drm_display_mode *mode = &crtc->config.adjusted_mode; + enum pipe pipe = crtc->pipe; + int position, vtotal; - if (INTEL_INFO(dev)->gen < 7) { - status = pipe == PIPE_A ? - DE_PIPEA_VBLANK : - DE_PIPEB_VBLANK; - } else { - switch (pipe) { - default: - case PIPE_A: - status = DE_PIPEA_VBLANK_IVB; - break; - case PIPE_B: - status = DE_PIPEB_VBLANK_IVB; - break; - case PIPE_C: - status = DE_PIPEC_VBLANK_IVB; - break; - } - } + vtotal = mode->crtc_vtotal; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + vtotal /= 2; + + if (IS_GEN2(dev)) + position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; + else + position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; - return __raw_i915_read32(dev_priv, DEISR) & status; + /* + * See update_scanline_offset() for the details on the + * scanline_offset adjustment. + */ + return (position + crtc->scanline_offset) % vtotal; } static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, @@ -657,7 +910,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, struct intel_crtc *intel_crtc = to_intel_crtc(crtc); const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode; int position; - int vbl_start, vbl_end, htotal, vtotal; + int vbl_start, vbl_end, hsync_start, htotal, vtotal; bool in_vbl = true; int ret = 0; unsigned long irqflags; @@ -669,6 +922,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, } htotal = mode->crtc_htotal; + hsync_start = mode->crtc_hsync_start; vtotal = mode->crtc_vtotal; vbl_start = mode->crtc_vblank_start; vbl_end = mode->crtc_vblank_end; @@ -687,7 +941,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, * following code must not block on uncore.lock. */ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - + /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */ /* Get optional system timestamp before query. */ @@ -698,47 +952,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, /* No obvious pixelcount register. Only query vertical * scanout position from Display scan line register. */ - if (IS_GEN2(dev)) - position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; - else - position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; - - if (HAS_PCH_SPLIT(dev)) { - /* - * The scanline counter increments at the leading edge - * of hsync, ie. it completely misses the active portion - * of the line. Fix up the counter at both edges of vblank - * to get a more accurate picture whether we're in vblank - * or not. - */ - in_vbl = ilk_pipe_in_vblank_locked(dev, pipe); - if ((in_vbl && position == vbl_start - 1) || - (!in_vbl && position == vbl_end - 1)) - position = (position + 1) % vtotal; - } else { - /* - * ISR vblank status bits don't work the way we'd want - * them to work on non-PCH platforms (for - * ilk_pipe_in_vblank_locked()), and there doesn't - * appear any other way to determine if we're currently - * in vblank. - * - * Instead let's assume that we're already in vblank if - * we got called from the vblank interrupt and the - * scanline counter value indicates that we're on the - * line just prior to vblank start. This should result - * in the correct answer, unless the vblank interrupt - * delivery really got delayed for almost exactly one - * full frame/field. - */ - if (flags & DRM_CALLED_FROM_VBLIRQ && - position == vbl_start - 1) { - position = (position + 1) % vtotal; - - /* Signal this correction as "applied". */ - ret |= 0x8; - } - } + position = __intel_get_crtc_scanline(intel_crtc); } else { /* Have access to pixelcount since start of frame. * We can split this into vertical and horizontal @@ -750,6 +964,29 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, vbl_start *= htotal; vbl_end *= htotal; vtotal *= htotal; + + /* + * In interlaced modes, the pixel counter counts all pixels, + * so one field will have htotal more pixels. In order to avoid + * the reported position from jumping backwards when the pixel + * counter is beyond the length of the shorter field, just + * clamp the position the length of the shorter field. This + * matches how the scanline counter based position works since + * the scanline counter doesn't count the two half lines. + */ + if (position >= vtotal) + position = vtotal - 1; + + /* + * Start of vblank interrupt is triggered at start of hsync, + * just prior to the first active line of vblank. However we + * consider lines to start at the leading edge of horizontal + * active. So, should we get here before we've crossed into + * the horizontal active of the first line in vblank, we would + * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that, + * always add htotal-hsync_start to the current pixel position. + */ + position = (position + htotal - hsync_start) % vtotal; } /* Get optional system timestamp after query. */ @@ -788,6 +1025,19 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, return ret; } +int intel_get_crtc_scanline(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + unsigned long irqflags; + int position; + + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + position = __intel_get_crtc_scanline(crtc); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + + return position; +} + static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe, int *max_error, struct timeval *vblank_time, @@ -833,7 +1083,7 @@ static bool intel_hpd_irq_event(struct drm_device *dev, DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n", connector->base.id, - drm_get_connector_name(connector), + connector->name, drm_get_connector_status_name(old_status), drm_get_connector_status_name(connector->status)); @@ -847,8 +1097,8 @@ static bool intel_hpd_irq_event(struct drm_device *dev, static void i915_hotplug_work_func(struct work_struct *work) { - drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t, - hotplug_work); + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, hotplug_work); struct drm_device *dev = dev_priv->dev; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_connector *intel_connector; @@ -878,7 +1128,7 @@ static void i915_hotplug_work_func(struct work_struct *work) connector->polled == DRM_CONNECTOR_POLL_HPD) { DRM_INFO("HPD interrupt storm detected on connector %s: " "switching from hotplug detection to polling\n", - drm_get_connector_name(connector)); + connector->name); dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED; connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT; @@ -886,7 +1136,7 @@ static void i915_hotplug_work_func(struct work_struct *work) } if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) { DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n", - drm_get_connector_name(connector), intel_encoder->hpd_pin); + connector->name, intel_encoder->hpd_pin); } } /* if there were no outputs to poll, poll was disabled, @@ -916,9 +1166,14 @@ static void i915_hotplug_work_func(struct work_struct *work) drm_kms_helper_hotplug_event(dev); } +static void intel_hpd_irq_uninstall(struct drm_i915_private *dev_priv) +{ + del_timer_sync(&dev_priv->hotplug_reenable_timer); +} + static void ironlake_rps_change_irq_handler(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 busy_up, busy_down, max_avg, min_avg; u8 new_delay; @@ -956,9 +1211,9 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev) } static void notify_ring(struct drm_device *dev, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { - if (ring->obj == NULL) + if (!intel_ring_initialized(ring)) return; trace_i915_gem_request_complete(ring); @@ -969,22 +1224,26 @@ static void notify_ring(struct drm_device *dev, static void gen6_pm_rps_work(struct work_struct *work) { - drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t, - rps.work); + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, rps.work); u32 pm_iir; int new_delay, adj; spin_lock_irq(&dev_priv->irq_lock); pm_iir = dev_priv->rps.pm_iir; dev_priv->rps.pm_iir = 0; - /* Make sure not to corrupt PMIMR state used by ringbuffer code */ - snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS); + if (IS_BROADWELL(dev_priv->dev)) + bdw_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + else { + /* Make sure not to corrupt PMIMR state used by ringbuffer */ + snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + } spin_unlock_irq(&dev_priv->irq_lock); /* Make sure we didn't queue anything we're not going to process. */ - WARN_ON(pm_iir & ~GEN6_PM_RPS_EVENTS); + WARN_ON(pm_iir & ~dev_priv->pm_rps_events); - if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0) + if ((pm_iir & dev_priv->pm_rps_events) == 0) return; mutex_lock(&dev_priv->rps.hw_lock); @@ -995,36 +1254,38 @@ static void gen6_pm_rps_work(struct work_struct *work) adj *= 2; else adj = 1; - new_delay = dev_priv->rps.cur_delay + adj; + new_delay = dev_priv->rps.cur_freq + adj; /* * For better performance, jump directly * to RPe if we're below it. */ - if (new_delay < dev_priv->rps.rpe_delay) - new_delay = dev_priv->rps.rpe_delay; + if (new_delay < dev_priv->rps.efficient_freq) + new_delay = dev_priv->rps.efficient_freq; } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { - if (dev_priv->rps.cur_delay > dev_priv->rps.rpe_delay) - new_delay = dev_priv->rps.rpe_delay; + if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq) + new_delay = dev_priv->rps.efficient_freq; else - new_delay = dev_priv->rps.min_delay; + new_delay = dev_priv->rps.min_freq_softlimit; adj = 0; } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { if (adj < 0) adj *= 2; else adj = -1; - new_delay = dev_priv->rps.cur_delay + adj; + new_delay = dev_priv->rps.cur_freq + adj; } else { /* unknown event */ - new_delay = dev_priv->rps.cur_delay; + new_delay = dev_priv->rps.cur_freq; } /* sysfs frequency interfaces may have snuck in while servicing the * interrupt */ new_delay = clamp_t(int, new_delay, - dev_priv->rps.min_delay, dev_priv->rps.max_delay); - dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_delay; + dev_priv->rps.min_freq_softlimit, + dev_priv->rps.max_freq_softlimit); + + dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq; if (IS_VALLEYVIEW(dev_priv->dev)) valleyview_set_rps(dev_priv->dev, new_delay); @@ -1046,8 +1307,8 @@ static void gen6_pm_rps_work(struct work_struct *work) */ static void ivybridge_parity_work(struct work_struct *work) { - drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t, - l3_parity.error_work); + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, l3_parity.error_work); u32 error_status, row, bank, subbank; char *parity_event[6]; uint32_t misccpctl; @@ -1119,7 +1380,7 @@ out: static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; if (!HAS_L3_DPF(dev)) return; @@ -1165,14 +1426,27 @@ static void snb_gt_irq_handler(struct drm_device *dev, if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | GT_BSD_CS_ERROR_INTERRUPT | GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) { - DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir); - i915_handle_error(dev, false); + i915_handle_error(dev, false, "GT error interrupt 0x%08x", + gt_iir); } if (gt_iir & GT_PARITY_ERROR(dev)) ivybridge_parity_error_irq_handler(dev, gt_iir); } +static void gen8_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) +{ + if ((pm_iir & dev_priv->pm_rps_events) == 0) + return; + + spin_lock(&dev_priv->irq_lock); + dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; + bdw_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); + spin_unlock(&dev_priv->irq_lock); + + queue_work(dev_priv->wq, &dev_priv->rps.work); +} + static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev, struct drm_i915_private *dev_priv, u32 master_ctl) @@ -1196,18 +1470,32 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev, DRM_ERROR("The master control interrupt lied (GT0)!\n"); } - if (master_ctl & GEN8_GT_VCS1_IRQ) { + if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) { tmp = I915_READ(GEN8_GT_IIR(1)); if (tmp) { ret = IRQ_HANDLED; vcs = tmp >> GEN8_VCS1_IRQ_SHIFT; if (vcs & GT_RENDER_USER_INTERRUPT) notify_ring(dev, &dev_priv->ring[VCS]); + vcs = tmp >> GEN8_VCS2_IRQ_SHIFT; + if (vcs & GT_RENDER_USER_INTERRUPT) + notify_ring(dev, &dev_priv->ring[VCS2]); I915_WRITE(GEN8_GT_IIR(1), tmp); } else DRM_ERROR("The master control interrupt lied (GT1)!\n"); } + if (master_ctl & GEN8_GT_PM_IRQ) { + tmp = I915_READ(GEN8_GT_IIR(2)); + if (tmp & dev_priv->pm_rps_events) { + ret = IRQ_HANDLED; + gen8_rps_irq_handler(dev_priv, tmp); + I915_WRITE(GEN8_GT_IIR(2), + tmp & dev_priv->pm_rps_events); + } else + DRM_ERROR("The master control interrupt lied (PM)!\n"); + } + if (master_ctl & GEN8_GT_VECS_IRQ) { tmp = I915_READ(GEN8_GT_IIR(3)); if (tmp) { @@ -1230,20 +1518,33 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, const u32 *hpd) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int i; bool storm_detected = false; if (!hotplug_trigger) return; + DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", + hotplug_trigger); + spin_lock(&dev_priv->irq_lock); for (i = 1; i < HPD_NUM_PINS; i++) { - WARN_ONCE(hpd[i] & hotplug_trigger && - dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED, - "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n", - hotplug_trigger, i, hpd[i]); + if (hpd[i] & hotplug_trigger && + dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) { + /* + * On GMCH platforms the interrupt mask bits only + * prevent irq generation, not the setting of the + * hotplug bits itself. So only WARN about unexpected + * interrupts on saner platforms. + */ + WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev), + "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n", + hotplug_trigger, i, hpd[i]); + + continue; + } if (!(hpd[i] & hotplug_trigger) || dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED) @@ -1283,14 +1584,14 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev, static void gmbus_irq_handler(struct drm_device *dev) { - struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; wake_up_all(&dev_priv->gmbus_wait_queue); } static void dp_aux_irq_handler(struct drm_device *dev) { - struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; wake_up_all(&dev_priv->gmbus_wait_queue); } @@ -1396,10 +1697,10 @@ static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) * the work queue. */ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) { - if (pm_iir & GEN6_PM_RPS_EVENTS) { + if (pm_iir & dev_priv->pm_rps_events) { spin_lock(&dev_priv->irq_lock); - dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS; - snb_disable_pm_irq(dev_priv, pm_iir & GEN6_PM_RPS_EVENTS); + dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; + snb_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); spin_unlock(&dev_priv->irq_lock); queue_work(dev_priv->wq, &dev_priv->rps.work); @@ -1410,23 +1711,132 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) notify_ring(dev_priv->dev, &dev_priv->ring[VECS]); if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) { - DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir); - i915_handle_error(dev_priv->dev, false); + i915_handle_error(dev_priv->dev, false, + "VEBOX CS error interrupt 0x%08x", + pm_iir); + } + } +} + +static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe) +{ + struct intel_crtc *crtc; + + if (!drm_handle_vblank(dev, pipe)) + return false; + + crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + wake_up(&crtc->vbl_wait); + + return true; +} + +static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 pipe_stats[I915_MAX_PIPES] = { }; + int pipe; + + spin_lock(&dev_priv->irq_lock); + for_each_pipe(pipe) { + int reg; + u32 mask, iir_bit = 0; + + /* + * PIPESTAT bits get signalled even when the interrupt is + * disabled with the mask bits, and some of the status bits do + * not generate interrupts at all (like the underrun bit). Hence + * we need to be careful that we only handle what we want to + * handle. + */ + mask = 0; + if (__cpu_fifo_underrun_reporting_enabled(dev, pipe)) + mask |= PIPE_FIFO_UNDERRUN_STATUS; + + switch (pipe) { + case PIPE_A: + iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT; + break; + case PIPE_B: + iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; + break; + case PIPE_C: + iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; + break; + } + if (iir & iir_bit) + mask |= dev_priv->pipestat_irq_mask[pipe]; + + if (!mask) + continue; + + reg = PIPESTAT(pipe); + mask |= PIPESTAT_INT_ENABLE_MASK; + pipe_stats[pipe] = I915_READ(reg) & mask; + + /* + * Clear the PIPE*STAT regs before the IIR + */ + if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS | + PIPESTAT_INT_STATUS_MASK)) + I915_WRITE(reg, pipe_stats[pipe]); + } + spin_unlock(&dev_priv->irq_lock); + + for_each_pipe(pipe) { + if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) + intel_pipe_handle_vblank(dev, pipe); + + if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) { + intel_prepare_page_flip(dev, pipe); + intel_finish_page_flip(dev, pipe); } + + if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) + i9xx_pipe_crc_irq_handler(dev, pipe); + + if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS && + intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) + DRM_ERROR("pipe %c underrun\n", pipe_name(pipe)); } + + if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) + gmbus_irq_handler(dev); +} + +static void i9xx_hpd_irq_handler(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); + + if (IS_G4X(dev)) { + u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; + + intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_g4x); + } else { + u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; + + intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915); + } + + if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) && + hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) + dp_aux_irq_handler(dev); + + I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); + /* + * Make sure hotplug status is cleared before we clear IIR, or else we + * may miss hotplug events. + */ + POSTING_READ(PORT_HOTPLUG_STAT); } static irqreturn_t valleyview_irq_handler(int irq, void *arg) { - struct drm_device *dev = (struct drm_device *) arg; - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_device *dev = arg; + struct drm_i915_private *dev_priv = dev->dev_private; u32 iir, gt_iir, pm_iir; irqreturn_t ret = IRQ_NONE; - unsigned long irqflags; - int pipe; - u32 pipe_stats[I915_MAX_PIPES]; - - atomic_inc(&dev_priv->irq_received); while (true) { iir = I915_READ(VLV_IIR); @@ -1440,71 +1850,61 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg) snb_gt_irq_handler(dev, dev_priv, gt_iir); - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - for_each_pipe(pipe) { - int reg = PIPESTAT(pipe); - pipe_stats[pipe] = I915_READ(reg); + valleyview_pipestat_irq_handler(dev, iir); - /* - * Clear the PIPE*STAT regs before the IIR - */ - if (pipe_stats[pipe] & 0x8000ffff) { - if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) - DRM_DEBUG_DRIVER("pipe %c underrun\n", - pipe_name(pipe)); - I915_WRITE(reg, pipe_stats[pipe]); - } - } - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + /* Consume port. Then clear IIR or we'll miss events */ + if (iir & I915_DISPLAY_PORT_INTERRUPT) + i9xx_hpd_irq_handler(dev); - for_each_pipe(pipe) { - if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(dev, pipe); + if (pm_iir) + gen6_rps_irq_handler(dev_priv, pm_iir); - if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) { - intel_prepare_page_flip(dev, pipe); - intel_finish_page_flip(dev, pipe); - } + I915_WRITE(GTIIR, gt_iir); + I915_WRITE(GEN6_PMIIR, pm_iir); + I915_WRITE(VLV_IIR, iir); + } - if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) - i9xx_pipe_crc_irq_handler(dev, pipe); - } +out: + return ret; +} - /* Consume port. Then clear IIR or we'll miss events */ - if (iir & I915_DISPLAY_PORT_INTERRUPT) { - u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); - u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; +static irqreturn_t cherryview_irq_handler(int irq, void *arg) +{ + struct drm_device *dev = arg; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 master_ctl, iir; + irqreturn_t ret = IRQ_NONE; - DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", - hotplug_status); + for (;;) { + master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL; + iir = I915_READ(VLV_IIR); - intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915); + if (master_ctl == 0 && iir == 0) + break; - if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) - dp_aux_irq_handler(dev); + I915_WRITE(GEN8_MASTER_IRQ, 0); - I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); - I915_READ(PORT_HOTPLUG_STAT); - } + gen8_gt_irq_handler(dev, dev_priv, master_ctl); - if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) - gmbus_irq_handler(dev); + valleyview_pipestat_irq_handler(dev, iir); - if (pm_iir) - gen6_rps_irq_handler(dev_priv, pm_iir); + /* Consume port. Then clear IIR or we'll miss events */ + i9xx_hpd_irq_handler(dev); - I915_WRITE(GTIIR, gt_iir); - I915_WRITE(GEN6_PMIIR, pm_iir); I915_WRITE(VLV_IIR, iir); + + I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL); + POSTING_READ(GEN8_MASTER_IRQ); + + ret = IRQ_HANDLED; } -out: return ret; } static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; @@ -1547,12 +1947,12 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) if (pch_iir & SDE_TRANSA_FIFO_UNDER) if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, false)) - DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n"); + DRM_ERROR("PCH transcoder A FIFO underrun\n"); if (pch_iir & SDE_TRANSB_FIFO_UNDER) if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B, false)) - DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n"); + DRM_ERROR("PCH transcoder B FIFO underrun\n"); } static void ivb_err_int_handler(struct drm_device *dev) @@ -1568,8 +1968,8 @@ static void ivb_err_int_handler(struct drm_device *dev) if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) { if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) - DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n", - pipe_name(pipe)); + DRM_ERROR("Pipe %c FIFO underrun\n", + pipe_name(pipe)); } if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) { @@ -1594,24 +1994,24 @@ static void cpt_serr_int_handler(struct drm_device *dev) if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN) if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, false)) - DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n"); + DRM_ERROR("PCH transcoder A FIFO underrun\n"); if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN) if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B, false)) - DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n"); + DRM_ERROR("PCH transcoder B FIFO underrun\n"); if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN) if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C, false)) - DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n"); + DRM_ERROR("PCH transcoder C FIFO underrun\n"); I915_WRITE(SERR_INT, serr_int); } static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; @@ -1662,12 +2062,12 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir) for_each_pipe(pipe) { if (de_iir & DE_PIPE_VBLANK(pipe)) - drm_handle_vblank(dev, pipe); + intel_pipe_handle_vblank(dev, pipe); if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) - DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n", - pipe_name(pipe)); + DRM_ERROR("Pipe %c FIFO underrun\n", + pipe_name(pipe)); if (de_iir & DE_PIPE_CRC_DONE(pipe)) i9xx_pipe_crc_irq_handler(dev, pipe); @@ -1699,7 +2099,7 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir) static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) { struct drm_i915_private *dev_priv = dev->dev_private; - enum pipe i; + enum pipe pipe; if (de_iir & DE_ERR_INT_IVB) ivb_err_int_handler(dev); @@ -1710,14 +2110,14 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) if (de_iir & DE_GSE_IVB) intel_opregion_asle_intr(dev); - for_each_pipe(i) { - if (de_iir & (DE_PIPE_VBLANK_IVB(i))) - drm_handle_vblank(dev, i); + for_each_pipe(pipe) { + if (de_iir & (DE_PIPE_VBLANK_IVB(pipe))) + intel_pipe_handle_vblank(dev, pipe); /* plane/pipes map 1:1 on ilk+ */ - if (de_iir & DE_PLANE_FLIP_DONE_IVB(i)) { - intel_prepare_page_flip(dev, i); - intel_finish_page_flip_plane(dev, i); + if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) { + intel_prepare_page_flip(dev, pipe); + intel_finish_page_flip_plane(dev, pipe); } } @@ -1734,13 +2134,11 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) static irqreturn_t ironlake_irq_handler(int irq, void *arg) { - struct drm_device *dev = (struct drm_device *) arg; - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_device *dev = arg; + struct drm_i915_private *dev_priv = dev->dev_private; u32 de_iir, gt_iir, de_ier, sde_ier = 0; irqreturn_t ret = IRQ_NONE; - atomic_inc(&dev_priv->irq_received); - /* We get interrupts on unclaimed registers, so check for this before we * do any I915_{READ,WRITE}. */ intel_uncore_check_errors(dev); @@ -1809,8 +2207,6 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) uint32_t tmp = 0; enum pipe pipe; - atomic_inc(&dev_priv->irq_received); - master_ctl = I915_READ(GEN8_MASTER_IRQ); master_ctl &= ~GEN8_MASTER_IRQ_CONTROL; if (!master_ctl) @@ -1859,9 +2255,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe)); if (pipe_iir & GEN8_PIPE_VBLANK) - drm_handle_vblank(dev, pipe); + intel_pipe_handle_vblank(dev, pipe); - if (pipe_iir & GEN8_PIPE_FLIP_DONE) { + if (pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE) { intel_prepare_page_flip(dev, pipe); intel_finish_page_flip_plane(dev, pipe); } @@ -1872,8 +2268,8 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) { if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) - DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n", - pipe_name(pipe)); + DRM_ERROR("Pipe %c FIFO underrun\n", + pipe_name(pipe)); } if (pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS) { @@ -1914,7 +2310,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) static void i915_error_wake_up(struct drm_i915_private *dev_priv, bool reset_completed) { - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; int i; /* @@ -1950,8 +2346,8 @@ static void i915_error_work_func(struct work_struct *work) { struct i915_gpu_error *error = container_of(work, struct i915_gpu_error, work); - drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t, - gpu_error); + struct drm_i915_private *dev_priv = + container_of(error, struct drm_i915_private, gpu_error); struct drm_device *dev = dev_priv->dev; char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; @@ -1976,6 +2372,14 @@ static void i915_error_work_func(struct work_struct *work) reset_event); /* + * In most cases it's guaranteed that we get here with an RPM + * reference held, for example because there is a pending GPU + * request that won't finish until the reset is done. This + * isn't the case at least when we get here by doing a + * simulated reset via debugs, so get an RPM reference. + */ + intel_runtime_pm_get(dev_priv); + /* * All state reset _must_ be completed before we update the * reset counter, for otherwise waiters might miss the reset * pending state and not properly drop locks, resulting in @@ -1985,6 +2389,8 @@ static void i915_error_work_func(struct work_struct *work) intel_display_handle_reset(dev); + intel_runtime_pm_put(dev_priv); + if (ret == 0) { /* * After all the gem state is reset, increment the reset @@ -1996,7 +2402,7 @@ static void i915_error_work_func(struct work_struct *work) * updates before * the counter increment. */ - smp_mb__before_atomic_inc(); + smp_mb__before_atomic(); atomic_inc(&dev_priv->gpu_error.reset_counter); kobject_uevent_env(&dev->primary->kdev->kobj, @@ -2115,11 +2521,18 @@ static void i915_report_and_clear_eir(struct drm_device *dev) * so userspace knows something bad happened (should trigger collection * of a ring dump etc.). */ -void i915_handle_error(struct drm_device *dev, bool wedged) +void i915_handle_error(struct drm_device *dev, bool wedged, + const char *fmt, ...) { struct drm_i915_private *dev_priv = dev->dev_private; + va_list args; + char error_msg[80]; + + va_start(args, fmt); + vscnprintf(error_msg, sizeof(error_msg), fmt, args); + va_end(args); - i915_capture_error_state(dev); + i915_capture_error_state(dev, wedged, error_msg); i915_report_and_clear_eir(dev); if (wedged) { @@ -2153,7 +2566,7 @@ void i915_handle_error(struct drm_device *dev, bool wedged) static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_i915_gem_object *obj; @@ -2185,8 +2598,8 @@ static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, in } else { int dspaddr = DSPADDR(intel_crtc->plane); stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) + - crtc->y * crtc->fb->pitches[0] + - crtc->x * crtc->fb->bits_per_pixel/8); + crtc->y * crtc->primary->fb->pitches[0] + + crtc->x * crtc->primary->fb->bits_per_pixel/8); } spin_unlock_irqrestore(&dev->event_lock, flags); @@ -2202,7 +2615,7 @@ static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, in */ static int i915_enable_vblank(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; if (!i915_pipe_enabled(dev, pipe)) @@ -2211,14 +2624,10 @@ static int i915_enable_vblank(struct drm_device *dev, int pipe) spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (INTEL_INFO(dev)->gen >= 4) i915_enable_pipestat(dev_priv, pipe, - PIPE_START_VBLANK_INTERRUPT_ENABLE); + PIPE_START_VBLANK_INTERRUPT_STATUS); else i915_enable_pipestat(dev_priv, pipe, - PIPE_VBLANK_INTERRUPT_ENABLE); - - /* maintain vblank delivery even in deep C-states */ - if (dev_priv->info->gen == 3) - I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS)); + PIPE_VBLANK_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); return 0; @@ -2226,7 +2635,7 @@ static int i915_enable_vblank(struct drm_device *dev, int pipe) static int ironlake_enable_vblank(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); @@ -2243,22 +2652,15 @@ static int ironlake_enable_vblank(struct drm_device *dev, int pipe) static int valleyview_enable_vblank(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; - u32 imr; if (!i915_pipe_enabled(dev, pipe)) return -EINVAL; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - imr = I915_READ(VLV_IMR); - if (pipe == PIPE_A) - imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT; - else - imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; - I915_WRITE(VLV_IMR, imr); i915_enable_pipestat(dev_priv, pipe, - PIPE_START_VBLANK_INTERRUPT_ENABLE); + PIPE_START_VBLANK_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); return 0; @@ -2285,22 +2687,19 @@ static int gen8_enable_vblank(struct drm_device *dev, int pipe) */ static void i915_disable_vblank(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - if (dev_priv->info->gen == 3) - I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS)); - i915_disable_pipestat(dev_priv, pipe, - PIPE_VBLANK_INTERRUPT_ENABLE | - PIPE_START_VBLANK_INTERRUPT_ENABLE); + PIPE_VBLANK_INTERRUPT_STATUS | + PIPE_START_VBLANK_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } static void ironlake_disable_vblank(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); @@ -2312,19 +2711,12 @@ static void ironlake_disable_vblank(struct drm_device *dev, int pipe) static void valleyview_disable_vblank(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; - u32 imr; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); i915_disable_pipestat(dev_priv, pipe, - PIPE_START_VBLANK_INTERRUPT_ENABLE); - imr = I915_READ(VLV_IMR); - if (pipe == PIPE_A) - imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT; - else - imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; - I915_WRITE(VLV_IMR, imr); + PIPE_START_VBLANK_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } @@ -2344,80 +2736,149 @@ static void gen8_disable_vblank(struct drm_device *dev, int pipe) } static u32 -ring_last_seqno(struct intel_ring_buffer *ring) +ring_last_seqno(struct intel_engine_cs *ring) { return list_entry(ring->request_list.prev, struct drm_i915_gem_request, list)->seqno; } static bool -ring_idle(struct intel_ring_buffer *ring, u32 seqno) +ring_idle(struct intel_engine_cs *ring, u32 seqno) { return (list_empty(&ring->request_list) || i915_seqno_passed(seqno, ring_last_seqno(ring))); } -static struct intel_ring_buffer * -semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno) +static bool +ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr) +{ + if (INTEL_INFO(dev)->gen >= 8) { + /* + * FIXME: gen8 semaphore support - currently we don't emit + * semaphores on bdw anyway, but this needs to be addressed when + * we merge that code. + */ + return false; + } else { + ipehr &= ~MI_SEMAPHORE_SYNC_MASK; + return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | + MI_SEMAPHORE_REGISTER); + } +} + +static struct intel_engine_cs * +semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr) +{ + struct drm_i915_private *dev_priv = ring->dev->dev_private; + struct intel_engine_cs *signaller; + int i; + + if (INTEL_INFO(dev_priv->dev)->gen >= 8) { + /* + * FIXME: gen8 semaphore support - currently we don't emit + * semaphores on bdw anyway, but this needs to be addressed when + * we merge that code. + */ + return NULL; + } else { + u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; + + for_each_ring(signaller, dev_priv, i) { + if(ring == signaller) + continue; + + if (sync_bits == signaller->semaphore.mbox.wait[ring->id]) + return signaller; + } + } + + DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x\n", + ring->id, ipehr); + + return NULL; +} + +static struct intel_engine_cs * +semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno) { struct drm_i915_private *dev_priv = ring->dev->dev_private; - u32 cmd, ipehr, acthd, acthd_min; + u32 cmd, ipehr, head; + int i; ipehr = I915_READ(RING_IPEHR(ring->mmio_base)); - if ((ipehr & ~(0x3 << 16)) != - (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER)) + if (!ipehr_is_semaphore_wait(ring->dev, ipehr)) return NULL; - /* ACTHD is likely pointing to the dword after the actual command, - * so scan backwards until we find the MBOX. + /* + * HEAD is likely pointing to the dword after the actual command, + * so scan backwards until we find the MBOX. But limit it to just 3 + * dwords. Note that we don't care about ACTHD here since that might + * point at at batch, and semaphores are always emitted into the + * ringbuffer itself. */ - acthd = intel_ring_get_active_head(ring) & HEAD_ADDR; - acthd_min = max((int)acthd - 3 * 4, 0); - do { - cmd = ioread32(ring->virtual_start + acthd); + head = I915_READ_HEAD(ring) & HEAD_ADDR; + + for (i = 4; i; --i) { + /* + * Be paranoid and presume the hw has gone off into the wild - + * our ring is smaller than what the hardware (and hence + * HEAD_ADDR) allows. Also handles wrap-around. + */ + head &= ring->buffer->size - 1; + + /* This here seems to blow up */ + cmd = ioread32(ring->buffer->virtual_start + head); if (cmd == ipehr) break; - acthd -= 4; - if (acthd < acthd_min) - return NULL; - } while (1); + head -= 4; + } + + if (!i) + return NULL; - *seqno = ioread32(ring->virtual_start+acthd+4)+1; - return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3]; + *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1; + return semaphore_wait_to_signaller_ring(ring, ipehr); } -static int semaphore_passed(struct intel_ring_buffer *ring) +static int semaphore_passed(struct intel_engine_cs *ring) { struct drm_i915_private *dev_priv = ring->dev->dev_private; - struct intel_ring_buffer *signaller; - u32 seqno, ctl; + struct intel_engine_cs *signaller; + u32 seqno; - ring->hangcheck.deadlock = true; + ring->hangcheck.deadlock++; signaller = semaphore_waits_for(ring, &seqno); - if (signaller == NULL || signaller->hangcheck.deadlock) + if (signaller == NULL) return -1; + /* Prevent pathological recursion due to driver bugs */ + if (signaller->hangcheck.deadlock >= I915_NUM_RINGS) + return -1; + + if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno)) + return 1; + /* cursory check for an unkickable deadlock */ - ctl = I915_READ_CTL(signaller); - if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0) + if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && + semaphore_passed(signaller) < 0) return -1; - return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno); + return 0; } static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) { - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; int i; for_each_ring(ring, dev_priv, i) - ring->hangcheck.deadlock = false; + ring->hangcheck.deadlock = 0; } static enum intel_ring_hangcheck_action -ring_stuck(struct intel_ring_buffer *ring, u32 acthd) +ring_stuck(struct intel_engine_cs *ring, u64 acthd) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -2436,9 +2897,9 @@ ring_stuck(struct intel_ring_buffer *ring, u32 acthd) */ tmp = I915_READ_CTL(ring); if (tmp & RING_WAIT) { - DRM_ERROR("Kicking stuck wait on %s\n", - ring->name); - i915_handle_error(dev, false); + i915_handle_error(dev, false, + "Kicking stuck wait on %s", + ring->name); I915_WRITE_CTL(ring, tmp); return HANGCHECK_KICK; } @@ -2448,9 +2909,9 @@ ring_stuck(struct intel_ring_buffer *ring, u32 acthd) default: return HANGCHECK_HUNG; case 1: - DRM_ERROR("Kicking stuck semaphore on %s\n", - ring->name); - i915_handle_error(dev, false); + i915_handle_error(dev, false, + "Kicking stuck semaphore on %s", + ring->name); I915_WRITE_CTL(ring, tmp); return HANGCHECK_KICK; case 0: @@ -2472,21 +2933,21 @@ ring_stuck(struct intel_ring_buffer *ring, u32 acthd) static void i915_hangcheck_elapsed(unsigned long data) { struct drm_device *dev = (struct drm_device *)data; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring; int i; int busy_count = 0, rings_hung = 0; bool stuck[I915_NUM_RINGS] = { 0 }; #define BUSY 1 #define KICK 5 #define HUNG 20 -#define FIRE 30 - if (!i915_enable_hangcheck) + if (!i915.enable_hangcheck) return; for_each_ring(ring, dev_priv, i) { - u32 seqno, acthd; + u64 acthd; + u32 seqno; bool busy = true; semaphore_clear_deadlocks(dev_priv); @@ -2564,7 +3025,7 @@ static void i915_hangcheck_elapsed(unsigned long data) } for_each_ring(ring, dev_priv, i) { - if (ring->hangcheck.score > FIRE) { + if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { DRM_INFO("%s on %s\n", stuck[i] ? "stuck" : "no progress", ring->name); @@ -2573,7 +3034,7 @@ static void i915_hangcheck_elapsed(unsigned long data) } if (rings_hung) - return i915_handle_error(dev, true); + return i915_handle_error(dev, true, "Ring hung"); if (busy_count) /* Reset timer case chip hangs without another request @@ -2584,75 +3045,77 @@ static void i915_hangcheck_elapsed(unsigned long data) void i915_queue_hangcheck(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (!i915_enable_hangcheck) + if (!i915.enable_hangcheck) return; mod_timer(&dev_priv->gpu_error.hangcheck_timer, round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES)); } -static void ibx_irq_preinstall(struct drm_device *dev) +static void ibx_irq_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; if (HAS_PCH_NOP(dev)) return; - /* south display irq */ - I915_WRITE(SDEIMR, 0xffffffff); - /* - * SDEIER is also touched by the interrupt handler to work around missed - * PCH interrupts. Hence we can't update it after the interrupt handler - * is enabled - instead we unconditionally enable all PCH interrupt - * sources here, but then only unmask them as needed with SDEIMR. - */ + GEN5_IRQ_RESET(SDE); + + if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev)) + I915_WRITE(SERR_INT, 0xffffffff); +} + +/* + * SDEIER is also touched by the interrupt handler to work around missed PCH + * interrupts. Hence we can't update it after the interrupt handler is enabled - + * instead we unconditionally enable all PCH interrupt sources here, but then + * only unmask them as needed with SDEIMR. + * + * This function needs to be called before interrupts are enabled. + */ +static void ibx_irq_pre_postinstall(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (HAS_PCH_NOP(dev)) + return; + + WARN_ON(I915_READ(SDEIER) != 0); I915_WRITE(SDEIER, 0xffffffff); POSTING_READ(SDEIER); } -static void gen5_gt_irq_preinstall(struct drm_device *dev) +static void gen5_gt_irq_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - /* and GT */ - I915_WRITE(GTIMR, 0xffffffff); - I915_WRITE(GTIER, 0x0); - POSTING_READ(GTIER); - - if (INTEL_INFO(dev)->gen >= 6) { - /* and PM */ - I915_WRITE(GEN6_PMIMR, 0xffffffff); - I915_WRITE(GEN6_PMIER, 0x0); - POSTING_READ(GEN6_PMIER); - } + GEN5_IRQ_RESET(GT); + if (INTEL_INFO(dev)->gen >= 6) + GEN5_IRQ_RESET(GEN6_PM); } /* drm_dma.h hooks */ -static void ironlake_irq_preinstall(struct drm_device *dev) +static void ironlake_irq_reset(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - - atomic_set(&dev_priv->irq_received, 0); + struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(HWSTAM, 0xeffe); + I915_WRITE(HWSTAM, 0xffffffff); - I915_WRITE(DEIMR, 0xffffffff); - I915_WRITE(DEIER, 0x0); - POSTING_READ(DEIER); + GEN5_IRQ_RESET(DE); + if (IS_GEN7(dev)) + I915_WRITE(GEN7_ERR_INT, 0xffffffff); - gen5_gt_irq_preinstall(dev); + gen5_gt_irq_reset(dev); - ibx_irq_preinstall(dev); + ibx_irq_reset(dev); } static void valleyview_irq_preinstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - atomic_set(&dev_priv->irq_received, 0); - /* VLV magic */ I915_WRITE(VLV_IMR, 0); I915_WRITE(RING_IMR(RENDER_RING_BASE), 0); @@ -2663,7 +3126,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev) I915_WRITE(GTIIR, I915_READ(GTIIR)); I915_WRITE(GTIIR, I915_READ(GTIIR)); - gen5_gt_irq_preinstall(dev); + gen5_gt_irq_reset(dev); I915_WRITE(DPINVGTT, 0xff); @@ -2677,58 +3140,65 @@ static void valleyview_irq_preinstall(struct drm_device *dev) POSTING_READ(VLV_IER); } -static void gen8_irq_preinstall(struct drm_device *dev) +static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv) +{ + GEN8_IRQ_RESET_NDX(GT, 0); + GEN8_IRQ_RESET_NDX(GT, 1); + GEN8_IRQ_RESET_NDX(GT, 2); + GEN8_IRQ_RESET_NDX(GT, 3); +} + +static void gen8_irq_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - atomic_set(&dev_priv->irq_received, 0); - I915_WRITE(GEN8_MASTER_IRQ, 0); POSTING_READ(GEN8_MASTER_IRQ); - /* IIR can theoretically queue up two events. Be paranoid */ -#define GEN8_IRQ_INIT_NDX(type, which) do { \ - I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \ - POSTING_READ(GEN8_##type##_IMR(which)); \ - I915_WRITE(GEN8_##type##_IER(which), 0); \ - I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ - POSTING_READ(GEN8_##type##_IIR(which)); \ - I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ - } while (0) - -#define GEN8_IRQ_INIT(type) do { \ - I915_WRITE(GEN8_##type##_IMR, 0xffffffff); \ - POSTING_READ(GEN8_##type##_IMR); \ - I915_WRITE(GEN8_##type##_IER, 0); \ - I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \ - POSTING_READ(GEN8_##type##_IIR); \ - I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \ - } while (0) - - GEN8_IRQ_INIT_NDX(GT, 0); - GEN8_IRQ_INIT_NDX(GT, 1); - GEN8_IRQ_INIT_NDX(GT, 2); - GEN8_IRQ_INIT_NDX(GT, 3); + gen8_gt_irq_reset(dev_priv); - for_each_pipe(pipe) { - GEN8_IRQ_INIT_NDX(DE_PIPE, pipe); - } + for_each_pipe(pipe) + GEN8_IRQ_RESET_NDX(DE_PIPE, pipe); - GEN8_IRQ_INIT(DE_PORT); - GEN8_IRQ_INIT(DE_MISC); - GEN8_IRQ_INIT(PCU); -#undef GEN8_IRQ_INIT -#undef GEN8_IRQ_INIT_NDX + GEN5_IRQ_RESET(GEN8_DE_PORT_); + GEN5_IRQ_RESET(GEN8_DE_MISC_); + GEN5_IRQ_RESET(GEN8_PCU_); + + ibx_irq_reset(dev); +} + +static void cherryview_irq_preinstall(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe; + + I915_WRITE(GEN8_MASTER_IRQ, 0); + POSTING_READ(GEN8_MASTER_IRQ); + + gen8_gt_irq_reset(dev_priv); + + GEN5_IRQ_RESET(GEN8_PCU_); POSTING_READ(GEN8_PCU_IIR); - ibx_irq_preinstall(dev); + I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV); + + I915_WRITE(PORT_HOTPLUG_EN, 0); + I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); + + for_each_pipe(pipe) + I915_WRITE(PIPESTAT(pipe), 0xffff); + + I915_WRITE(VLV_IMR, 0xffffffff); + I915_WRITE(VLV_IER, 0x0); + I915_WRITE(VLV_IIR, 0xffffffff); + POSTING_READ(VLV_IIR); } static void ibx_hpd_irq_setup(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *intel_encoder; u32 hotplug_irqs, hotplug, enabled_irqs = 0; @@ -2763,22 +3233,18 @@ static void ibx_hpd_irq_setup(struct drm_device *dev) static void ibx_irq_postinstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 mask; if (HAS_PCH_NOP(dev)) return; - if (HAS_PCH_IBX(dev)) { - mask = SDE_GMBUS | SDE_AUX_MASK | SDE_TRANSB_FIFO_UNDER | - SDE_TRANSA_FIFO_UNDER | SDE_POISON; - } else { - mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT | SDE_ERROR_CPT; - - I915_WRITE(SERR_INT, I915_READ(SERR_INT)); - } + if (HAS_PCH_IBX(dev)) + mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON; + else + mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT; - I915_WRITE(SDEIIR, I915_READ(SDEIIR)); + GEN5_ASSERT_IIR_IS_ZERO(SDEIIR); I915_WRITE(SDEIMR, ~mask); } @@ -2804,58 +3270,49 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; } - I915_WRITE(GTIIR, I915_READ(GTIIR)); - I915_WRITE(GTIMR, dev_priv->gt_irq_mask); - I915_WRITE(GTIER, gt_irqs); - POSTING_READ(GTIER); + GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs); if (INTEL_INFO(dev)->gen >= 6) { - pm_irqs |= GEN6_PM_RPS_EVENTS; + pm_irqs |= dev_priv->pm_rps_events; if (HAS_VEBOX(dev)) pm_irqs |= PM_VEBOX_USER_INTERRUPT; dev_priv->pm_irq_mask = 0xffffffff; - I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR)); - I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask); - I915_WRITE(GEN6_PMIER, pm_irqs); - POSTING_READ(GEN6_PMIER); + GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs); } } static int ironlake_irq_postinstall(struct drm_device *dev) { unsigned long irqflags; - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 display_mask, extra_mask; if (INTEL_INFO(dev)->gen >= 7) { display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB | DE_PLANEB_FLIP_DONE_IVB | - DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB | - DE_ERR_INT_IVB); + DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB); extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB | - DE_PIPEA_VBLANK_IVB); - - I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT)); + DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB); } else { display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE | DE_AUX_CHANNEL_A | - DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN | DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE | DE_POISON); - extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT; + extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT | + DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN; } dev_priv->irq_mask = ~display_mask; - /* should always can generate irq */ - I915_WRITE(DEIIR, I915_READ(DEIIR)); - I915_WRITE(DEIMR, dev_priv->irq_mask); - I915_WRITE(DEIER, display_mask | extra_mask); - POSTING_READ(DEIER); + I915_WRITE(HWSTAM, 0xeffe); + + ibx_irq_pre_postinstall(dev); + + GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask); gen5_gt_irq_postinstall(dev); @@ -2875,44 +3332,113 @@ static int ironlake_irq_postinstall(struct drm_device *dev) return 0; } +static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv) +{ + u32 pipestat_mask; + u32 iir_mask; + + pipestat_mask = PIPESTAT_INT_STATUS_MASK | + PIPE_FIFO_UNDERRUN_STATUS; + + I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask); + I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask); + POSTING_READ(PIPESTAT(PIPE_A)); + + pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV | + PIPE_CRC_DONE_INTERRUPT_STATUS; + + i915_enable_pipestat(dev_priv, PIPE_A, pipestat_mask | + PIPE_GMBUS_INTERRUPT_STATUS); + i915_enable_pipestat(dev_priv, PIPE_B, pipestat_mask); + + iir_mask = I915_DISPLAY_PORT_INTERRUPT | + I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | + I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; + dev_priv->irq_mask &= ~iir_mask; + + I915_WRITE(VLV_IIR, iir_mask); + I915_WRITE(VLV_IIR, iir_mask); + I915_WRITE(VLV_IMR, dev_priv->irq_mask); + I915_WRITE(VLV_IER, ~dev_priv->irq_mask); + POSTING_READ(VLV_IER); +} + +static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv) +{ + u32 pipestat_mask; + u32 iir_mask; + + iir_mask = I915_DISPLAY_PORT_INTERRUPT | + I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | + I915_DISPLAY_PIPE_B_EVENT_INTERRUPT; + + dev_priv->irq_mask |= iir_mask; + I915_WRITE(VLV_IER, ~dev_priv->irq_mask); + I915_WRITE(VLV_IMR, dev_priv->irq_mask); + I915_WRITE(VLV_IIR, iir_mask); + I915_WRITE(VLV_IIR, iir_mask); + POSTING_READ(VLV_IIR); + + pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV | + PIPE_CRC_DONE_INTERRUPT_STATUS; + + i915_disable_pipestat(dev_priv, PIPE_A, pipestat_mask | + PIPE_GMBUS_INTERRUPT_STATUS); + i915_disable_pipestat(dev_priv, PIPE_B, pipestat_mask); + + pipestat_mask = PIPESTAT_INT_STATUS_MASK | + PIPE_FIFO_UNDERRUN_STATUS; + I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask); + I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask); + POSTING_READ(PIPESTAT(PIPE_A)); +} + +void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv) +{ + assert_spin_locked(&dev_priv->irq_lock); + + if (dev_priv->display_irqs_enabled) + return; + + dev_priv->display_irqs_enabled = true; + + if (dev_priv->dev->irq_enabled) + valleyview_display_irqs_install(dev_priv); +} + +void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv) +{ + assert_spin_locked(&dev_priv->irq_lock); + + if (!dev_priv->display_irqs_enabled) + return; + + dev_priv->display_irqs_enabled = false; + + if (dev_priv->dev->irq_enabled) + valleyview_display_irqs_uninstall(dev_priv); +} + static int valleyview_irq_postinstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - u32 enable_mask; - u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV | - PIPE_CRC_DONE_ENABLE; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; - enable_mask = I915_DISPLAY_PORT_INTERRUPT; - enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | - I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT | - I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | - I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; - - /* - *Leave vblank interrupts masked initially. enable/disable will - * toggle them based on usage. - */ - dev_priv->irq_mask = (~enable_mask) | - I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT | - I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT; + dev_priv->irq_mask = ~0; I915_WRITE(PORT_HOTPLUG_EN, 0); POSTING_READ(PORT_HOTPLUG_EN); I915_WRITE(VLV_IMR, dev_priv->irq_mask); - I915_WRITE(VLV_IER, enable_mask); + I915_WRITE(VLV_IER, ~dev_priv->irq_mask); I915_WRITE(VLV_IIR, 0xffffffff); - I915_WRITE(PIPESTAT(0), 0xffff); - I915_WRITE(PIPESTAT(1), 0xffff); POSTING_READ(VLV_IER); /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - i915_enable_pipestat(dev_priv, PIPE_A, pipestat_enable); - i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_EVENT_ENABLE); - i915_enable_pipestat(dev_priv, PIPE_B, pipestat_enable); + if (dev_priv->display_irqs_enabled) + valleyview_display_irqs_install(dev_priv); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); I915_WRITE(VLV_IIR, 0xffffffff); @@ -2946,49 +3472,38 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT }; - for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++) { - u32 tmp = I915_READ(GEN8_GT_IIR(i)); - if (tmp) - DRM_ERROR("Interrupt (%d) should have been masked in pre-install 0x%08x\n", - i, tmp); - I915_WRITE(GEN8_GT_IMR(i), ~gt_interrupts[i]); - I915_WRITE(GEN8_GT_IER(i), gt_interrupts[i]); - } - POSTING_READ(GEN8_GT_IER(0)); + for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++) + GEN8_IRQ_INIT_NDX(GT, i, ~gt_interrupts[i], gt_interrupts[i]); + + dev_priv->pm_irq_mask = 0xffffffff; } static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; - uint32_t de_pipe_masked = GEN8_PIPE_FLIP_DONE | + uint32_t de_pipe_masked = GEN8_PIPE_PRIMARY_FLIP_DONE | GEN8_PIPE_CDCLK_CRC_DONE | - GEN8_PIPE_FIFO_UNDERRUN | GEN8_DE_PIPE_IRQ_FAULT_ERRORS; - uint32_t de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK; + uint32_t de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK | + GEN8_PIPE_FIFO_UNDERRUN; int pipe; dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked; dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked; dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked; - for_each_pipe(pipe) { - u32 tmp = I915_READ(GEN8_DE_PIPE_IIR(pipe)); - if (tmp) - DRM_ERROR("Interrupt (%d) should have been masked in pre-install 0x%08x\n", - pipe, tmp); - I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]); - I915_WRITE(GEN8_DE_PIPE_IER(pipe), de_pipe_enables); - } - POSTING_READ(GEN8_DE_PIPE_ISR(0)); + for_each_pipe(pipe) + GEN8_IRQ_INIT_NDX(DE_PIPE, pipe, dev_priv->de_irq_mask[pipe], + de_pipe_enables); - I915_WRITE(GEN8_DE_PORT_IMR, ~GEN8_AUX_CHANNEL_A); - I915_WRITE(GEN8_DE_PORT_IER, GEN8_AUX_CHANNEL_A); - POSTING_READ(GEN8_DE_PORT_IER); + GEN5_IRQ_INIT(GEN8_DE_PORT_, ~GEN8_AUX_CHANNEL_A, GEN8_AUX_CHANNEL_A); } static int gen8_irq_postinstall(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + ibx_irq_pre_postinstall(dev); + gen8_gt_irq_postinstall(dev_priv); gen8_de_irq_postinstall(dev_priv); @@ -3000,57 +3515,69 @@ static int gen8_irq_postinstall(struct drm_device *dev) return 0; } -static void gen8_irq_uninstall(struct drm_device *dev) +static int cherryview_irq_postinstall(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + u32 enable_mask = I915_DISPLAY_PORT_INTERRUPT | + I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | + I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | + I915_DISPLAY_PIPE_C_EVENT_INTERRUPT; + u32 pipestat_enable = PLANE_FLIP_DONE_INT_STATUS_VLV | + PIPE_CRC_DONE_INTERRUPT_STATUS; + unsigned long irqflags; int pipe; - if (!dev_priv) - return; + /* + * Leave vblank interrupts masked initially. enable/disable will + * toggle them based on usage. + */ + dev_priv->irq_mask = ~enable_mask; - atomic_set(&dev_priv->irq_received, 0); + for_each_pipe(pipe) + I915_WRITE(PIPESTAT(pipe), 0xffff); - I915_WRITE(GEN8_MASTER_IRQ, 0); + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); + for_each_pipe(pipe) + i915_enable_pipestat(dev_priv, pipe, pipestat_enable); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); -#define GEN8_IRQ_FINI_NDX(type, which) do { \ - I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \ - I915_WRITE(GEN8_##type##_IER(which), 0); \ - I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ - } while (0) + I915_WRITE(VLV_IIR, 0xffffffff); + I915_WRITE(VLV_IMR, dev_priv->irq_mask); + I915_WRITE(VLV_IER, enable_mask); -#define GEN8_IRQ_FINI(type) do { \ - I915_WRITE(GEN8_##type##_IMR, 0xffffffff); \ - I915_WRITE(GEN8_##type##_IER, 0); \ - I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \ - } while (0) + gen8_gt_irq_postinstall(dev_priv); - GEN8_IRQ_FINI_NDX(GT, 0); - GEN8_IRQ_FINI_NDX(GT, 1); - GEN8_IRQ_FINI_NDX(GT, 2); - GEN8_IRQ_FINI_NDX(GT, 3); + I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE); + POSTING_READ(GEN8_MASTER_IRQ); - for_each_pipe(pipe) { - GEN8_IRQ_FINI_NDX(DE_PIPE, pipe); - } + return 0; +} - GEN8_IRQ_FINI(DE_PORT); - GEN8_IRQ_FINI(DE_MISC); - GEN8_IRQ_FINI(PCU); -#undef GEN8_IRQ_FINI -#undef GEN8_IRQ_FINI_NDX +static void gen8_irq_uninstall(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; - POSTING_READ(GEN8_PCU_IIR); + if (!dev_priv) + return; + + intel_hpd_irq_uninstall(dev_priv); + + gen8_irq_reset(dev); } static void valleyview_irq_uninstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long irqflags; int pipe; if (!dev_priv) return; - del_timer_sync(&dev_priv->hotplug_reenable_timer); + I915_WRITE(VLV_MASTER_IER, 0); + + intel_hpd_irq_uninstall(dev_priv); for_each_pipe(pipe) I915_WRITE(PIPESTAT(pipe), 0xffff); @@ -3058,52 +3585,88 @@ static void valleyview_irq_uninstall(struct drm_device *dev) I915_WRITE(HWSTAM, 0xffffffff); I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); - for_each_pipe(pipe) - I915_WRITE(PIPESTAT(pipe), 0xffff); + + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + if (dev_priv->display_irqs_enabled) + valleyview_display_irqs_uninstall(dev_priv); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + + dev_priv->irq_mask = 0; + I915_WRITE(VLV_IIR, 0xffffffff); I915_WRITE(VLV_IMR, 0xffffffff); I915_WRITE(VLV_IER, 0x0); POSTING_READ(VLV_IER); } -static void ironlake_irq_uninstall(struct drm_device *dev) +static void cherryview_irq_uninstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe; if (!dev_priv) return; - del_timer_sync(&dev_priv->hotplug_reenable_timer); + I915_WRITE(GEN8_MASTER_IRQ, 0); + POSTING_READ(GEN8_MASTER_IRQ); - I915_WRITE(HWSTAM, 0xffffffff); +#define GEN8_IRQ_FINI_NDX(type, which) \ +do { \ + I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \ + I915_WRITE(GEN8_##type##_IER(which), 0); \ + I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ + POSTING_READ(GEN8_##type##_IIR(which)); \ + I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \ +} while (0) + +#define GEN8_IRQ_FINI(type) \ +do { \ + I915_WRITE(GEN8_##type##_IMR, 0xffffffff); \ + I915_WRITE(GEN8_##type##_IER, 0); \ + I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \ + POSTING_READ(GEN8_##type##_IIR); \ + I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \ +} while (0) - I915_WRITE(DEIMR, 0xffffffff); - I915_WRITE(DEIER, 0x0); - I915_WRITE(DEIIR, I915_READ(DEIIR)); - if (IS_GEN7(dev)) - I915_WRITE(GEN7_ERR_INT, I915_READ(GEN7_ERR_INT)); + GEN8_IRQ_FINI_NDX(GT, 0); + GEN8_IRQ_FINI_NDX(GT, 1); + GEN8_IRQ_FINI_NDX(GT, 2); + GEN8_IRQ_FINI_NDX(GT, 3); - I915_WRITE(GTIMR, 0xffffffff); - I915_WRITE(GTIER, 0x0); - I915_WRITE(GTIIR, I915_READ(GTIIR)); + GEN8_IRQ_FINI(PCU); - if (HAS_PCH_NOP(dev)) +#undef GEN8_IRQ_FINI +#undef GEN8_IRQ_FINI_NDX + + I915_WRITE(PORT_HOTPLUG_EN, 0); + I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); + + for_each_pipe(pipe) + I915_WRITE(PIPESTAT(pipe), 0xffff); + + I915_WRITE(VLV_IMR, 0xffffffff); + I915_WRITE(VLV_IER, 0x0); + I915_WRITE(VLV_IIR, 0xffffffff); + POSTING_READ(VLV_IIR); +} + +static void ironlake_irq_uninstall(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (!dev_priv) return; - I915_WRITE(SDEIMR, 0xffffffff); - I915_WRITE(SDEIER, 0x0); - I915_WRITE(SDEIIR, I915_READ(SDEIIR)); - if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev)) - I915_WRITE(SERR_INT, I915_READ(SERR_INT)); + intel_hpd_irq_uninstall(dev_priv); + + ironlake_irq_reset(dev); } static void i8xx_irq_preinstall(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - atomic_set(&dev_priv->irq_received, 0); - for_each_pipe(pipe) I915_WRITE(PIPESTAT(pipe), 0); I915_WRITE16(IMR, 0xffff); @@ -3113,7 +3676,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev) static int i8xx_irq_postinstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; I915_WRITE16(EMR, @@ -3138,8 +3701,8 @@ static int i8xx_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_ENABLE); - i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_ENABLE); + i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); + i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); return 0; @@ -3151,10 +3714,10 @@ static int i8xx_irq_postinstall(struct drm_device *dev) static bool i8xx_handle_vblank(struct drm_device *dev, int plane, int pipe, u32 iir) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); - if (!drm_handle_vblank(dev, pipe)) + if (!intel_pipe_handle_vblank(dev, pipe)) return false; if ((iir & flip_pending) == 0) @@ -3178,8 +3741,8 @@ static bool i8xx_handle_vblank(struct drm_device *dev, static irqreturn_t i8xx_irq_handler(int irq, void *arg) { - struct drm_device *dev = (struct drm_device *) arg; - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_device *dev = arg; + struct drm_i915_private *dev_priv = dev->dev_private; u16 iir, new_iir; u32 pipe_stats[2]; unsigned long irqflags; @@ -3188,8 +3751,6 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; - atomic_inc(&dev_priv->irq_received); - iir = I915_READ16(IIR); if (iir == 0) return IRQ_NONE; @@ -3202,7 +3763,9 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) - i915_handle_error(dev, false); + i915_handle_error(dev, false, + "Command parser error, iir 0x%08x", + iir); for_each_pipe(pipe) { int reg = PIPESTAT(pipe); @@ -3211,12 +3774,8 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) /* * Clear the PIPE*STAT regs before the IIR */ - if (pipe_stats[pipe] & 0x8000ffff) { - if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) - DRM_DEBUG_DRIVER("pipe %c underrun\n", - pipe_name(pipe)); + if (pipe_stats[pipe] & 0x8000ffff) I915_WRITE(reg, pipe_stats[pipe]); - } } spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); @@ -3239,6 +3798,10 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) i9xx_pipe_crc_irq_handler(dev, pipe); + + if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS && + intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) + DRM_ERROR("pipe %c underrun\n", pipe_name(pipe)); } iir = new_iir; @@ -3249,7 +3812,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) static void i8xx_irq_uninstall(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; for_each_pipe(pipe) { @@ -3264,11 +3827,9 @@ static void i8xx_irq_uninstall(struct drm_device * dev) static void i915_irq_preinstall(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - atomic_set(&dev_priv->irq_received, 0); - if (I915_HAS_HOTPLUG(dev)) { I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); @@ -3284,7 +3845,7 @@ static void i915_irq_preinstall(struct drm_device * dev) static int i915_irq_postinstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 enable_mask; unsigned long irqflags; @@ -3325,8 +3886,8 @@ static int i915_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_ENABLE); - i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_ENABLE); + i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); + i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); return 0; @@ -3338,10 +3899,10 @@ static int i915_irq_postinstall(struct drm_device *dev) static bool i915_handle_vblank(struct drm_device *dev, int plane, int pipe, u32 iir) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); - if (!drm_handle_vblank(dev, pipe)) + if (!intel_pipe_handle_vblank(dev, pipe)) return false; if ((iir & flip_pending) == 0) @@ -3365,8 +3926,8 @@ static bool i915_handle_vblank(struct drm_device *dev, static irqreturn_t i915_irq_handler(int irq, void *arg) { - struct drm_device *dev = (struct drm_device *) arg; - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_device *dev = arg; + struct drm_i915_private *dev_priv = dev->dev_private; u32 iir, new_iir, pipe_stats[I915_MAX_PIPES]; unsigned long irqflags; u32 flip_mask = @@ -3374,8 +3935,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; int pipe, ret = IRQ_NONE; - atomic_inc(&dev_priv->irq_received); - iir = I915_READ(IIR); do { bool irq_received = (iir & ~flip_mask) != 0; @@ -3388,7 +3947,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) - i915_handle_error(dev, false); + i915_handle_error(dev, false, + "Command parser error, iir 0x%08x", + iir); for_each_pipe(pipe) { int reg = PIPESTAT(pipe); @@ -3396,9 +3957,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) /* Clear the PIPE*STAT regs before the IIR */ if (pipe_stats[pipe] & 0x8000ffff) { - if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) - DRM_DEBUG_DRIVER("pipe %c underrun\n", - pipe_name(pipe)); I915_WRITE(reg, pipe_stats[pipe]); irq_received = true; } @@ -3409,19 +3967,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) break; /* Consume port. Then clear IIR or we'll miss events */ - if ((I915_HAS_HOTPLUG(dev)) && - (iir & I915_DISPLAY_PORT_INTERRUPT)) { - u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); - u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; - - DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", - hotplug_status); - - intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915); - - I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); - POSTING_READ(PORT_HOTPLUG_STAT); - } + if (I915_HAS_HOTPLUG(dev) && + iir & I915_DISPLAY_PORT_INTERRUPT) + i9xx_hpd_irq_handler(dev); I915_WRITE(IIR, iir & ~flip_mask); new_iir = I915_READ(IIR); /* Flush posted writes */ @@ -3443,6 +3991,10 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) i9xx_pipe_crc_irq_handler(dev, pipe); + + if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS && + intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) + DRM_ERROR("pipe %c underrun\n", pipe_name(pipe)); } if (blc_event || (iir & I915_ASLE_INTERRUPT)) @@ -3474,10 +4026,10 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) static void i915_irq_uninstall(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - del_timer_sync(&dev_priv->hotplug_reenable_timer); + intel_hpd_irq_uninstall(dev_priv); if (I915_HAS_HOTPLUG(dev)) { I915_WRITE(PORT_HOTPLUG_EN, 0); @@ -3498,11 +4050,9 @@ static void i915_irq_uninstall(struct drm_device * dev) static void i965_irq_preinstall(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; - atomic_set(&dev_priv->irq_received, 0); - I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); @@ -3516,7 +4066,7 @@ static void i965_irq_preinstall(struct drm_device * dev) static int i965_irq_postinstall(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 enable_mask; u32 error_mask; unsigned long irqflags; @@ -3541,9 +4091,9 @@ static int i965_irq_postinstall(struct drm_device *dev) /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_EVENT_ENABLE); - i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_ENABLE); - i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_ENABLE); + i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS); + i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS); + i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); /* @@ -3575,7 +4125,7 @@ static int i965_irq_postinstall(struct drm_device *dev) static void i915_hpd_irq_setup(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *intel_encoder; u32 hotplug_en; @@ -3606,26 +4156,22 @@ static void i915_hpd_irq_setup(struct drm_device *dev) static irqreturn_t i965_irq_handler(int irq, void *arg) { - struct drm_device *dev = (struct drm_device *) arg; - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_device *dev = arg; + struct drm_i915_private *dev_priv = dev->dev_private; u32 iir, new_iir; u32 pipe_stats[I915_MAX_PIPES]; unsigned long irqflags; - int irq_received; int ret = IRQ_NONE, pipe; u32 flip_mask = I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT; - atomic_inc(&dev_priv->irq_received); - iir = I915_READ(IIR); for (;;) { + bool irq_received = (iir & ~flip_mask) != 0; bool blc_event = false; - irq_received = (iir & ~flip_mask) != 0; - /* Can't rely on pipestat interrupt bit in iir as it might * have been cleared after the pipestat interrupt was received. * It doesn't set the bit in iir again, but it still produces @@ -3633,7 +4179,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) */ spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT) - i915_handle_error(dev, false); + i915_handle_error(dev, false, + "Command parser error, iir 0x%08x", + iir); for_each_pipe(pipe) { int reg = PIPESTAT(pipe); @@ -3643,11 +4191,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) * Clear the PIPE*STAT regs before the IIR */ if (pipe_stats[pipe] & 0x8000ffff) { - if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) - DRM_DEBUG_DRIVER("pipe %c underrun\n", - pipe_name(pipe)); I915_WRITE(reg, pipe_stats[pipe]); - irq_received = 1; + irq_received = true; } } spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); @@ -3658,25 +4203,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) ret = IRQ_HANDLED; /* Consume port. Then clear IIR or we'll miss events */ - if (iir & I915_DISPLAY_PORT_INTERRUPT) { - u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT); - u32 hotplug_trigger = hotplug_status & (IS_G4X(dev) ? - HOTPLUG_INT_STATUS_G4X : - HOTPLUG_INT_STATUS_I915); - - DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n", - hotplug_status); - - intel_hpd_irq_handler(dev, hotplug_trigger, - IS_G4X(dev) ? hpd_status_g4x : hpd_status_i915); - - if (IS_G4X(dev) && - (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)) - dp_aux_irq_handler(dev); - - I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status); - I915_READ(PORT_HOTPLUG_STAT); - } + if (iir & I915_DISPLAY_PORT_INTERRUPT) + i9xx_hpd_irq_handler(dev); I915_WRITE(IIR, iir & ~flip_mask); new_iir = I915_READ(IIR); /* Flush posted writes */ @@ -3696,8 +4224,11 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) i9xx_pipe_crc_irq_handler(dev, pipe); - } + if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS && + intel_set_cpu_fifo_underrun_reporting(dev, pipe, false)) + DRM_ERROR("pipe %c underrun\n", pipe_name(pipe)); + } if (blc_event || (iir & I915_ASLE_INTERRUPT)) intel_opregion_asle_intr(dev); @@ -3730,13 +4261,13 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) static void i965_irq_uninstall(struct drm_device * dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; int pipe; if (!dev_priv) return; - del_timer_sync(&dev_priv->hotplug_reenable_timer); + intel_hpd_irq_uninstall(dev_priv); I915_WRITE(PORT_HOTPLUG_EN, 0); I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); @@ -3753,9 +4284,9 @@ static void i965_irq_uninstall(struct drm_device * dev) I915_WRITE(IIR, I915_READ(IIR)); } -static void i915_reenable_hotplug_timer_func(unsigned long data) +static void intel_hpd_irq_reenable(unsigned long data) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *)data; + struct drm_i915_private *dev_priv = (struct drm_i915_private *)data; struct drm_device *dev = dev_priv->dev; struct drm_mode_config *mode_config = &dev->mode_config; unsigned long irqflags; @@ -3776,7 +4307,7 @@ static void i915_reenable_hotplug_timer_func(unsigned long data) if (intel_connector->encoder->hpd_pin == i) { if (connector->polled != intel_connector->polled) DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n", - drm_get_connector_name(connector)); + connector->name); connector->polled = intel_connector->polled; if (!connector->polled) connector->polled = DRM_CONNECTOR_POLL_HPD; @@ -3797,10 +4328,13 @@ void intel_irq_init(struct drm_device *dev) INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); + /* Let's track the enabled rps events */ + dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS; + setup_timer(&dev_priv->gpu_error.hangcheck_timer, i915_hangcheck_elapsed, (unsigned long) dev); - setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func, + setup_timer(&dev_priv->hotplug_reenable_timer, intel_hpd_irq_reenable, (unsigned long) dev_priv); pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); @@ -3821,7 +4355,15 @@ void intel_irq_init(struct drm_device *dev) dev->driver->get_scanout_position = i915_get_crtc_scanoutpos; } - if (IS_VALLEYVIEW(dev)) { + if (IS_CHERRYVIEW(dev)) { + dev->driver->irq_handler = cherryview_irq_handler; + dev->driver->irq_preinstall = cherryview_irq_preinstall; + dev->driver->irq_postinstall = cherryview_irq_postinstall; + dev->driver->irq_uninstall = cherryview_irq_uninstall; + dev->driver->enable_vblank = valleyview_enable_vblank; + dev->driver->disable_vblank = valleyview_disable_vblank; + dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; + } else if (IS_VALLEYVIEW(dev)) { dev->driver->irq_handler = valleyview_irq_handler; dev->driver->irq_preinstall = valleyview_irq_preinstall; dev->driver->irq_postinstall = valleyview_irq_postinstall; @@ -3831,7 +4373,7 @@ void intel_irq_init(struct drm_device *dev) dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; } else if (IS_GEN8(dev)) { dev->driver->irq_handler = gen8_irq_handler; - dev->driver->irq_preinstall = gen8_irq_preinstall; + dev->driver->irq_preinstall = gen8_irq_reset; dev->driver->irq_postinstall = gen8_irq_postinstall; dev->driver->irq_uninstall = gen8_irq_uninstall; dev->driver->enable_vblank = gen8_enable_vblank; @@ -3839,7 +4381,7 @@ void intel_irq_init(struct drm_device *dev) dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup; } else if (HAS_PCH_SPLIT(dev)) { dev->driver->irq_handler = ironlake_irq_handler; - dev->driver->irq_preinstall = ironlake_irq_preinstall; + dev->driver->irq_preinstall = ironlake_irq_reset; dev->driver->irq_postinstall = ironlake_irq_postinstall; dev->driver->irq_uninstall = ironlake_irq_uninstall; dev->driver->enable_vblank = ironlake_enable_vblank; @@ -3896,58 +4438,21 @@ void intel_hpd_init(struct drm_device *dev) spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -/* Disable interrupts so we can allow Package C8+. */ -void hsw_pc8_disable_interrupts(struct drm_device *dev) +/* Disable interrupts so we can allow runtime PM. */ +void intel_runtime_pm_disable_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - - dev_priv->pc8.regsave.deimr = I915_READ(DEIMR); - dev_priv->pc8.regsave.sdeimr = I915_READ(SDEIMR); - dev_priv->pc8.regsave.gtimr = I915_READ(GTIMR); - dev_priv->pc8.regsave.gtier = I915_READ(GTIER); - dev_priv->pc8.regsave.gen6_pmimr = I915_READ(GEN6_PMIMR); - - ironlake_disable_display_irq(dev_priv, 0xffffffff); - ibx_disable_display_interrupt(dev_priv, 0xffffffff); - ilk_disable_gt_irq(dev_priv, 0xffffffff); - snb_disable_pm_irq(dev_priv, 0xffffffff); - - dev_priv->pc8.irqs_disabled = true; - - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + dev->driver->irq_uninstall(dev); + dev_priv->pm.irqs_disabled = true; } -/* Restore interrupts so we can recover from Package C8+. */ -void hsw_pc8_restore_interrupts(struct drm_device *dev) +/* Restore interrupts so we can recover from runtime PM. */ +void intel_runtime_pm_restore_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long irqflags; - uint32_t val; - - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - - val = I915_READ(DEIMR); - WARN(val != 0xffffffff, "DEIMR is 0x%08x\n", val); - - val = I915_READ(SDEIMR); - WARN(val != 0xffffffff, "SDEIMR is 0x%08x\n", val); - val = I915_READ(GTIMR); - WARN(val != 0xffffffff, "GTIMR is 0x%08x\n", val); - - val = I915_READ(GEN6_PMIMR); - WARN(val != 0xffffffff, "GEN6_PMIMR is 0x%08x\n", val); - - dev_priv->pc8.irqs_disabled = false; - - ironlake_enable_display_irq(dev_priv, ~dev_priv->pc8.regsave.deimr); - ibx_enable_display_interrupt(dev_priv, ~dev_priv->pc8.regsave.sdeimr); - ilk_enable_gt_irq(dev_priv, ~dev_priv->pc8.regsave.gtimr); - snb_enable_pm_irq(dev_priv, ~dev_priv->pc8.regsave.gen6_pmimr); - I915_WRITE(GTIER, dev_priv->pc8.regsave.gtier); - - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + dev_priv->pm.irqs_disabled = false; + dev->driver->irq_preinstall(dev); + dev->driver->irq_postinstall(dev); } diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c new file mode 100644 index 00000000000..d05a2afa17d --- /dev/null +++ b/drivers/gpu/drm/i915/i915_params.c @@ -0,0 +1,158 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "i915_drv.h" + +struct i915_params i915 __read_mostly = { + .modeset = -1, + .panel_ignore_lid = 1, + .powersave = 1, + .semaphores = -1, + .lvds_downclock = 0, + .lvds_channel_mode = 0, + .panel_use_ssc = -1, + .vbt_sdvo_panel_type = -1, + .enable_rc6 = -1, + .enable_fbc = -1, + .enable_hangcheck = true, + .enable_ppgtt = -1, + .enable_psr = 0, + .preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT), + .disable_power_well = 1, + .enable_ips = 1, + .fastboot = 0, + .prefault_disable = 0, + .reset = true, + .invert_brightness = 0, + .disable_display = 0, + .enable_cmd_parser = 1, + .disable_vtd_wa = 0, +}; + +module_param_named(modeset, i915.modeset, int, 0400); +MODULE_PARM_DESC(modeset, + "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, " + "1=on, -1=force vga console preference [default])"); + +module_param_named(panel_ignore_lid, i915.panel_ignore_lid, int, 0600); +MODULE_PARM_DESC(panel_ignore_lid, + "Override lid status (0=autodetect, 1=autodetect disabled [default], " + "-1=force lid closed, -2=force lid open)"); + +module_param_named(powersave, i915.powersave, int, 0600); +MODULE_PARM_DESC(powersave, + "Enable powersavings, fbc, downclocking, etc. (default: true)"); + +module_param_named(semaphores, i915.semaphores, int, 0400); +MODULE_PARM_DESC(semaphores, + "Use semaphores for inter-ring sync " + "(default: -1 (use per-chip defaults))"); + +module_param_named(enable_rc6, i915.enable_rc6, int, 0400); +MODULE_PARM_DESC(enable_rc6, + "Enable power-saving render C-state 6. " + "Different stages can be selected via bitmask values " + "(0 = disable; 1 = enable rc6; 2 = enable deep rc6; 4 = enable deepest rc6). " + "For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. " + "default: -1 (use per-chip default)"); + +module_param_named(enable_fbc, i915.enable_fbc, int, 0600); +MODULE_PARM_DESC(enable_fbc, + "Enable frame buffer compression for power savings " + "(default: -1 (use per-chip default))"); + +module_param_named(lvds_downclock, i915.lvds_downclock, int, 0400); +MODULE_PARM_DESC(lvds_downclock, + "Use panel (LVDS/eDP) downclocking for power savings " + "(default: false)"); + +module_param_named(lvds_channel_mode, i915.lvds_channel_mode, int, 0600); +MODULE_PARM_DESC(lvds_channel_mode, + "Specify LVDS channel mode " + "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)"); + +module_param_named(lvds_use_ssc, i915.panel_use_ssc, int, 0600); +MODULE_PARM_DESC(lvds_use_ssc, + "Use Spread Spectrum Clock with panels [LVDS/eDP] " + "(default: auto from VBT)"); + +module_param_named(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0600); +MODULE_PARM_DESC(vbt_sdvo_panel_type, + "Override/Ignore selection of SDVO panel mode in the VBT " + "(-2=ignore, -1=auto [default], index in VBT BIOS table)"); + +module_param_named(reset, i915.reset, bool, 0600); +MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)"); + +module_param_named(enable_hangcheck, i915.enable_hangcheck, bool, 0644); +MODULE_PARM_DESC(enable_hangcheck, + "Periodically check GPU activity for detecting hangs. " + "WARNING: Disabling this can cause system wide hangs. " + "(default: true)"); + +module_param_named(enable_ppgtt, i915.enable_ppgtt, int, 0400); +MODULE_PARM_DESC(enable_ppgtt, + "Override PPGTT usage. " + "(-1=auto [default], 0=disabled, 1=aliasing, 2=full)"); + +module_param_named(enable_psr, i915.enable_psr, int, 0600); +MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)"); + +module_param_named(preliminary_hw_support, i915.preliminary_hw_support, int, 0600); +MODULE_PARM_DESC(preliminary_hw_support, + "Enable preliminary hardware support."); + +module_param_named(disable_power_well, i915.disable_power_well, int, 0600); +MODULE_PARM_DESC(disable_power_well, + "Disable the power well when possible (default: true)"); + +module_param_named(enable_ips, i915.enable_ips, int, 0600); +MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)"); + +module_param_named(fastboot, i915.fastboot, bool, 0600); +MODULE_PARM_DESC(fastboot, + "Try to skip unnecessary mode sets at boot time (default: false)"); + +module_param_named(prefault_disable, i915.prefault_disable, bool, 0600); +MODULE_PARM_DESC(prefault_disable, + "Disable page prefaulting for pread/pwrite/reloc (default:false). " + "For developers only."); + +module_param_named(invert_brightness, i915.invert_brightness, int, 0600); +MODULE_PARM_DESC(invert_brightness, + "Invert backlight brightness " + "(-1 force normal, 0 machine defaults, 1 force inversion), please " + "report PCI device ID, subsystem vendor and subsystem device ID " + "to dri-devel@lists.freedesktop.org, if your machine needs it. " + "It will then be included in an upcoming module version."); + +module_param_named(disable_display, i915.disable_display, bool, 0600); +MODULE_PARM_DESC(disable_display, "Disable display (default: false)"); + +module_param_named(disable_vtd_wa, i915.disable_vtd_wa, bool, 0600); +MODULE_PARM_DESC(disable_vtd_wa, "Disable all VT-d workarounds (default: false)"); + +module_param_named(enable_cmd_parser, i915.enable_cmd_parser, int, 0600); +MODULE_PARM_DESC(enable_cmd_parser, + "Enable command parsing (1=enabled [default], 0=disabled)"); diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index a48b7cad6f1..a5bab61bfc0 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -26,10 +26,11 @@ #define _I915_REG_H_ #define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a))) -#define _PIPE_INC(pipe, base, inc) ((base) + (pipe)*(inc)) #define _TRANSCODER(tran, a, b) ((a) + (tran)*((b)-(a))) #define _PORT(port, a, b) ((a) + (port)*((b)-(a))) +#define _PIPE3(pipe, a, b, c) (pipe < 2 ? _PIPE(pipe, a, b) : c) +#define _PORT3(port, a, b, c) (port < 2 ? _PORT(port, a, b) : c) #define _MASKED_BIT_ENABLE(a) (((a) << 16) | (a)) #define _MASKED_BIT_DISABLE(a) ((a) << 16) @@ -73,17 +74,24 @@ #define I915_GC_RENDER_CLOCK_166_MHZ (0 << 0) #define I915_GC_RENDER_CLOCK_200_MHZ (1 << 0) #define I915_GC_RENDER_CLOCK_333_MHZ (4 << 0) -#define LBB 0xf4 +#define PCI_LBPC 0xf4 /* legacy/combination backlight modes, also called LBB */ + /* Graphics reset regs */ #define I965_GDRST 0xc0 /* PCI config register */ -#define ILK_GDSR 0x2ca4 /* MCHBAR offset */ #define GRDOM_FULL (0<<2) #define GRDOM_RENDER (1<<2) #define GRDOM_MEDIA (3<<2) #define GRDOM_MASK (3<<2) #define GRDOM_RESET_ENABLE (1<<0) +#define ILK_GDSR 0x2ca4 /* MCHBAR offset */ +#define ILK_GRDOM_FULL (0<<1) +#define ILK_GRDOM_RENDER (1<<1) +#define ILK_GRDOM_MEDIA (3<<1) +#define ILK_GRDOM_MASK (3<<1) +#define ILK_GRDOM_RESET_ENABLE (1<<0) + #define GEN6_MBCUNIT_SNPCR 0x900c /* for LLC config */ #define GEN6_MBC_SNPCR_SHIFT 21 #define GEN6_MBC_SNPCR_MASK (3<<21) @@ -92,6 +100,9 @@ #define GEN6_MBC_SNPCR_LOW (2<<21) #define GEN6_MBC_SNPCR_MIN (3<<21) /* only 1/16th of the cache is shared */ +#define VLV_G3DCTL 0x9024 +#define VLV_GSCKGCTL 0x9028 + #define GEN6_MBCTL 0x0907c #define GEN6_MBCTL_ENABLE_BOOT_FETCH (1 << 4) #define GEN6_MBCTL_CTX_FETCH_NEEDED (1 << 3) @@ -175,9 +186,23 @@ #define VGA_CR_DATA_CGA 0x3d5 /* + * Instruction field definitions used by the command parser + */ +#define INSTR_CLIENT_SHIFT 29 +#define INSTR_CLIENT_MASK 0xE0000000 +#define INSTR_MI_CLIENT 0x0 +#define INSTR_BC_CLIENT 0x2 +#define INSTR_RC_CLIENT 0x3 +#define INSTR_SUBCLIENT_SHIFT 27 +#define INSTR_SUBCLIENT_MASK 0x18000000 +#define INSTR_MEDIA_SUBCLIENT 0x2 + +/* * Memory interface instructions used by the kernel */ #define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags)) +/* Many MI commands use bit 22 of the header dword for GGTT vs PPGTT */ +#define MI_GLOBAL_GTT (1<<22) #define MI_NOOP MI_INSTR(0, 0) #define MI_USER_INTERRUPT MI_INSTR(0x02, 0) @@ -232,7 +257,8 @@ #define MI_SEMAPHORE_SYNC_BVE (0<<16) /* VECS wait for BCS (VEBSYNC) */ #define MI_SEMAPHORE_SYNC_VVE (1<<16) /* VECS wait for VCS (VEVSYNC) */ #define MI_SEMAPHORE_SYNC_RVE (2<<16) /* VECS wait for RCS (VERSYNC) */ -#define MI_SEMAPHORE_SYNC_INVALID (3<<16) +#define MI_SEMAPHORE_SYNC_INVALID (3<<16) +#define MI_SEMAPHORE_SYNC_MASK (3<<16) #define MI_SET_CONTEXT MI_INSTR(0x18, 0) #define MI_MM_SPACE_GTT (1<<8) #define MI_MM_SPACE_PHYSICAL (0<<8) @@ -250,13 +276,16 @@ * - One can actually load arbitrary many arbitrary registers: Simply issue x * address/value pairs. Don't overdue it, though, x <= 2^4 must hold! */ -#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1) -#define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*x-1) +#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*(x)-1) +#define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*(x)-1) +#define MI_STORE_REGISTER_MEM_GEN8(x) MI_INSTR(0x24, 3*(x)-1) #define MI_SRM_LRM_GLOBAL_GTT (1<<22) #define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */ #define MI_FLUSH_DW_STORE_INDEX (1<<21) #define MI_INVALIDATE_TLB (1<<18) #define MI_FLUSH_DW_OP_STOREDW (1<<14) +#define MI_FLUSH_DW_OP_MASK (3<<14) +#define MI_FLUSH_DW_NOTIFY (1<<8) #define MI_INVALIDATE_BSD (1<<7) #define MI_FLUSH_DW_USE_GTT (1<<2) #define MI_FLUSH_DW_USE_PPGTT (0<<2) @@ -318,9 +347,12 @@ #define DISPLAY_PLANE_B (1<<20) #define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|(len-2)) #define PIPE_CONTROL_GLOBAL_GTT_IVB (1<<24) /* gen7+ */ +#define PIPE_CONTROL_MMIO_WRITE (1<<23) +#define PIPE_CONTROL_STORE_DATA_INDEX (1<<21) #define PIPE_CONTROL_CS_STALL (1<<20) #define PIPE_CONTROL_TLB_INVALIDATE (1<<18) #define PIPE_CONTROL_QW_WRITE (1<<14) +#define PIPE_CONTROL_POST_SYNC_OP_MASK (3<<14) #define PIPE_CONTROL_DEPTH_STALL (1<<13) #define PIPE_CONTROL_WRITE_FLUSH (1<<12) #define PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH (1<<12) /* gen6+ */ @@ -335,6 +367,94 @@ #define PIPE_CONTROL_DEPTH_CACHE_FLUSH (1<<0) #define PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */ +/* + * Commands used only by the command parser + */ +#define MI_SET_PREDICATE MI_INSTR(0x01, 0) +#define MI_ARB_CHECK MI_INSTR(0x05, 0) +#define MI_RS_CONTROL MI_INSTR(0x06, 0) +#define MI_URB_ATOMIC_ALLOC MI_INSTR(0x09, 0) +#define MI_PREDICATE MI_INSTR(0x0C, 0) +#define MI_RS_CONTEXT MI_INSTR(0x0F, 0) +#define MI_TOPOLOGY_FILTER MI_INSTR(0x0D, 0) +#define MI_LOAD_SCAN_LINES_EXCL MI_INSTR(0x13, 0) +#define MI_URB_CLEAR MI_INSTR(0x19, 0) +#define MI_UPDATE_GTT MI_INSTR(0x23, 0) +#define MI_CLFLUSH MI_INSTR(0x27, 0) +#define MI_REPORT_PERF_COUNT MI_INSTR(0x28, 0) +#define MI_REPORT_PERF_COUNT_GGTT (1<<0) +#define MI_LOAD_REGISTER_MEM MI_INSTR(0x29, 0) +#define MI_LOAD_REGISTER_REG MI_INSTR(0x2A, 0) +#define MI_RS_STORE_DATA_IMM MI_INSTR(0x2B, 0) +#define MI_LOAD_URB_MEM MI_INSTR(0x2C, 0) +#define MI_STORE_URB_MEM MI_INSTR(0x2D, 0) +#define MI_CONDITIONAL_BATCH_BUFFER_END MI_INSTR(0x36, 0) + +#define PIPELINE_SELECT ((0x3<<29)|(0x1<<27)|(0x1<<24)|(0x4<<16)) +#define GFX_OP_3DSTATE_VF_STATISTICS ((0x3<<29)|(0x1<<27)|(0x0<<24)|(0xB<<16)) +#define MEDIA_VFE_STATE ((0x3<<29)|(0x2<<27)|(0x0<<24)|(0x0<<16)) +#define MEDIA_VFE_STATE_MMIO_ACCESS_MASK (0x18) +#define GPGPU_OBJECT ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x4<<16)) +#define GPGPU_WALKER ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x5<<16)) +#define GFX_OP_3DSTATE_DX9_CONSTANTF_VS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x39<<16)) +#define GFX_OP_3DSTATE_DX9_CONSTANTF_PS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x3A<<16)) +#define GFX_OP_3DSTATE_SO_DECL_LIST \ + ((0x3<<29)|(0x3<<27)|(0x1<<24)|(0x17<<16)) + +#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x43<<16)) +#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x44<<16)) +#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x45<<16)) +#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x46<<16)) +#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS \ + ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x47<<16)) + +#define MFX_WAIT ((0x3<<29)|(0x1<<27)|(0x0<<16)) + +#define COLOR_BLT ((0x2<<29)|(0x40<<22)) +#define SRC_COPY_BLT ((0x2<<29)|(0x43<<22)) + +/* + * Registers used only by the command parser + */ +#define BCS_SWCTRL 0x22200 + +#define HS_INVOCATION_COUNT 0x2300 +#define DS_INVOCATION_COUNT 0x2308 +#define IA_VERTICES_COUNT 0x2310 +#define IA_PRIMITIVES_COUNT 0x2318 +#define VS_INVOCATION_COUNT 0x2320 +#define GS_INVOCATION_COUNT 0x2328 +#define GS_PRIMITIVES_COUNT 0x2330 +#define CL_INVOCATION_COUNT 0x2338 +#define CL_PRIMITIVES_COUNT 0x2340 +#define PS_INVOCATION_COUNT 0x2348 +#define PS_DEPTH_COUNT 0x2350 + +/* There are the 4 64-bit counter registers, one for each stream output */ +#define GEN7_SO_NUM_PRIMS_WRITTEN(n) (0x5200 + (n) * 8) + +#define GEN7_SO_PRIM_STORAGE_NEEDED(n) (0x5240 + (n) * 8) + +#define GEN7_3DPRIM_END_OFFSET 0x2420 +#define GEN7_3DPRIM_START_VERTEX 0x2430 +#define GEN7_3DPRIM_VERTEX_COUNT 0x2434 +#define GEN7_3DPRIM_INSTANCE_COUNT 0x2438 +#define GEN7_3DPRIM_START_INSTANCE 0x243C +#define GEN7_3DPRIM_BASE_VERTEX 0x2440 + +#define OACONTROL 0x2360 + +#define _GEN7_PIPEA_DE_LOAD_SL 0x70068 +#define _GEN7_PIPEB_DE_LOAD_SL 0x71068 +#define GEN7_PIPE_DE_LOAD_SL(pipe) _PIPE(pipe, \ + _GEN7_PIPEA_DE_LOAD_SL, \ + _GEN7_PIPEB_DE_LOAD_SL) /* * Reset registers @@ -358,6 +478,7 @@ #define IOSF_PORT_PUNIT 0x4 #define IOSF_PORT_NC 0x11 #define IOSF_PORT_DPIO 0x12 +#define IOSF_PORT_DPIO_2 0x1a #define IOSF_PORT_GPIO_NC 0x13 #define IOSF_PORT_CCK 0x14 #define IOSF_PORT_CCU 0xA9 @@ -369,22 +490,35 @@ /* See configdb bunit SB addr map */ #define BUNIT_REG_BISOC 0x11 -#define PUNIT_OPCODE_REG_READ 6 -#define PUNIT_OPCODE_REG_WRITE 7 - #define PUNIT_REG_DSPFREQ 0x36 #define DSPFREQSTAT_SHIFT 30 #define DSPFREQSTAT_MASK (0x3 << DSPFREQSTAT_SHIFT) #define DSPFREQGUAR_SHIFT 14 #define DSPFREQGUAR_MASK (0x3 << DSPFREQGUAR_SHIFT) + +/* See the PUNIT HAS v0.8 for the below bits */ +enum punit_power_well { + PUNIT_POWER_WELL_RENDER = 0, + PUNIT_POWER_WELL_MEDIA = 1, + PUNIT_POWER_WELL_DISP2D = 3, + PUNIT_POWER_WELL_DPIO_CMN_BC = 5, + PUNIT_POWER_WELL_DPIO_TX_B_LANES_01 = 6, + PUNIT_POWER_WELL_DPIO_TX_B_LANES_23 = 7, + PUNIT_POWER_WELL_DPIO_TX_C_LANES_01 = 8, + PUNIT_POWER_WELL_DPIO_TX_C_LANES_23 = 9, + PUNIT_POWER_WELL_DPIO_RX0 = 10, + PUNIT_POWER_WELL_DPIO_RX1 = 11, + + PUNIT_POWER_WELL_NUM, +}; + #define PUNIT_REG_PWRGT_CTRL 0x60 #define PUNIT_REG_PWRGT_STATUS 0x61 -#define PUNIT_CLK_GATE 1 -#define PUNIT_PWR_RESET 2 -#define PUNIT_PWR_GATE 3 -#define RENDER_PWRGT (PUNIT_PWR_GATE << 0) -#define MEDIA_PWRGT (PUNIT_PWR_GATE << 2) -#define DISP2D_PWRGT (PUNIT_PWR_GATE << 6) +#define PUNIT_PWRGT_MASK(power_well) (3 << ((power_well) * 2)) +#define PUNIT_PWRGT_PWR_ON(power_well) (0 << ((power_well) * 2)) +#define PUNIT_PWRGT_CLK_GATE(power_well) (1 << ((power_well) * 2)) +#define PUNIT_PWRGT_RESET(power_well) (2 << ((power_well) * 2)) +#define PUNIT_PWRGT_PWR_GATE(power_well) (3 << ((power_well) * 2)) #define PUNIT_REG_GPU_LFM 0xd3 #define PUNIT_REG_GPU_FREQ_REQ 0xd4 @@ -441,16 +575,91 @@ #define DSI_PLL_M1_DIV_MASK (0x1ff << 0) #define CCK_DISPLAY_CLOCK_CONTROL 0x6b -/* - * DPIO - a special bus for various display related registers to hide behind +/** + * DOC: DPIO * - * DPIO is VLV only. + * VLV and CHV have slightly peculiar display PHYs for driving DP/HDMI + * ports. DPIO is the name given to such a display PHY. These PHYs + * don't follow the standard programming model using direct MMIO + * registers, and instead their registers must be accessed trough IOSF + * sideband. VLV has one such PHY for driving ports B and C, and CHV + * adds another PHY for driving port D. Each PHY responds to specific + * IOSF-SB port. + * + * Each display PHY is made up of one or two channels. Each channel + * houses a common lane part which contains the PLL and other common + * logic. CH0 common lane also contains the IOSF-SB logic for the + * Common Register Interface (CRI) ie. the DPIO registers. CRI clock + * must be running when any DPIO registers are accessed. + * + * In addition to having their own registers, the PHYs are also + * controlled through some dedicated signals from the display + * controller. These include PLL reference clock enable, PLL enable, + * and CRI clock selection, for example. + * + * Eeach channel also has two splines (also called data lanes), and + * each spline is made up of one Physical Access Coding Sub-Layer + * (PCS) block and two TX lanes. So each channel has two PCS blocks + * and four TX lanes. The TX lanes are used as DP lanes or TMDS + * data/clock pairs depending on the output type. + * + * Additionally the PHY also contains an AUX lane with AUX blocks + * for each channel. This is used for DP AUX communication, but + * this fact isn't really relevant for the driver since AUX is + * controlled from the display controller side. No DPIO registers + * need to be accessed during AUX communication, + * + * Generally the common lane corresponds to the pipe and + * the spline (PCS/TX) correponds to the port. + * + * For dual channel PHY (VLV/CHV): + * + * pipe A == CMN/PLL/REF CH0 + * + * pipe B == CMN/PLL/REF CH1 + * + * port B == PCS/TX CH0 + * + * port C == PCS/TX CH1 + * + * This is especially important when we cross the streams + * ie. drive port B with pipe B, or port C with pipe A. + * + * For single channel PHY (CHV): + * + * pipe C == CMN/PLL/REF CH0 + * + * port D == PCS/TX CH0 + * + * Note: digital port B is DDI0, digital port C is DDI1, + * digital port D is DDI2 + */ +/* + * Dual channel PHY (VLV/CHV) + * --------------------------------- + * | CH0 | CH1 | + * | CMN/PLL/REF | CMN/PLL/REF | + * |---------------|---------------| Display PHY + * | PCS01 | PCS23 | PCS01 | PCS23 | + * |-------|-------|-------|-------| + * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| + * --------------------------------- + * | DDI0 | DDI1 | DP/HDMI ports + * --------------------------------- * - * Note: digital port B is DDI0, digital pot C is DDI1 + * Single channel PHY (CHV) + * ----------------- + * | CH0 | + * | CMN/PLL/REF | + * |---------------| Display PHY + * | PCS01 | PCS23 | + * |-------|-------| + * |TX0|TX1|TX2|TX3| + * ----------------- + * | DDI2 | DP/HDMI port + * ----------------- */ #define DPIO_DEVFN 0 -#define DPIO_OPCODE_REG_WRITE 1 -#define DPIO_OPCODE_REG_READ 0 #define DPIO_CTL (VLV_DISPLAY_BASE + 0x2110) #define DPIO_MODSEL1 (1<<3) /* if ref clk b == 27 */ @@ -527,14 +736,29 @@ #define DPIO_PCS_TX_LANE1_RESET (1<<7) #define VLV_PCS_DW0(ch) _PORT(ch, _VLV_PCS_DW0_CH0, _VLV_PCS_DW0_CH1) +#define _VLV_PCS01_DW0_CH0 0x200 +#define _VLV_PCS23_DW0_CH0 0x400 +#define _VLV_PCS01_DW0_CH1 0x2600 +#define _VLV_PCS23_DW0_CH1 0x2800 +#define VLV_PCS01_DW0(ch) _PORT(ch, _VLV_PCS01_DW0_CH0, _VLV_PCS01_DW0_CH1) +#define VLV_PCS23_DW0(ch) _PORT(ch, _VLV_PCS23_DW0_CH0, _VLV_PCS23_DW0_CH1) + #define _VLV_PCS_DW1_CH0 0x8204 #define _VLV_PCS_DW1_CH1 0x8404 +#define CHV_PCS_REQ_SOFTRESET_EN (1<<23) #define DPIO_PCS_CLK_CRI_RXEB_EIOS_EN (1<<22) #define DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN (1<<21) #define DPIO_PCS_CLK_DATAWIDTH_SHIFT (6) #define DPIO_PCS_CLK_SOFT_RESET (1<<5) #define VLV_PCS_DW1(ch) _PORT(ch, _VLV_PCS_DW1_CH0, _VLV_PCS_DW1_CH1) +#define _VLV_PCS01_DW1_CH0 0x204 +#define _VLV_PCS23_DW1_CH0 0x404 +#define _VLV_PCS01_DW1_CH1 0x2604 +#define _VLV_PCS23_DW1_CH1 0x2804 +#define VLV_PCS01_DW1(ch) _PORT(ch, _VLV_PCS01_DW1_CH0, _VLV_PCS01_DW1_CH1) +#define VLV_PCS23_DW1(ch) _PORT(ch, _VLV_PCS23_DW1_CH0, _VLV_PCS23_DW1_CH1) + #define _VLV_PCS_DW8_CH0 0x8220 #define _VLV_PCS_DW8_CH1 0x8420 #define VLV_PCS_DW8(ch) _PORT(ch, _VLV_PCS_DW8_CH0, _VLV_PCS_DW8_CH1) @@ -550,6 +774,19 @@ #define _VLV_PCS_DW9_CH1 0x8424 #define VLV_PCS_DW9(ch) _PORT(ch, _VLV_PCS_DW9_CH0, _VLV_PCS_DW9_CH1) +#define _CHV_PCS_DW10_CH0 0x8228 +#define _CHV_PCS_DW10_CH1 0x8428 +#define DPIO_PCS_SWING_CALC_TX0_TX2 (1<<30) +#define DPIO_PCS_SWING_CALC_TX1_TX3 (1<<31) +#define CHV_PCS_DW10(ch) _PORT(ch, _CHV_PCS_DW10_CH0, _CHV_PCS_DW10_CH1) + +#define _VLV_PCS01_DW10_CH0 0x0228 +#define _VLV_PCS23_DW10_CH0 0x0428 +#define _VLV_PCS01_DW10_CH1 0x2628 +#define _VLV_PCS23_DW10_CH1 0x2828 +#define VLV_PCS01_DW10(port) _PORT(port, _VLV_PCS01_DW10_CH0, _VLV_PCS01_DW10_CH1) +#define VLV_PCS23_DW10(port) _PORT(port, _VLV_PCS23_DW10_CH0, _VLV_PCS23_DW10_CH1) + #define _VLV_PCS_DW11_CH0 0x822c #define _VLV_PCS_DW11_CH1 0x842c #define VLV_PCS_DW11(ch) _PORT(ch, _VLV_PCS_DW11_CH0, _VLV_PCS_DW11_CH1) @@ -568,14 +805,21 @@ #define _VLV_TX_DW2_CH0 0x8288 #define _VLV_TX_DW2_CH1 0x8488 +#define DPIO_SWING_MARGIN_SHIFT 16 +#define DPIO_SWING_MARGIN_MASK (0xff << DPIO_SWING_MARGIN_SHIFT) +#define DPIO_UNIQ_TRANS_SCALE_SHIFT 8 #define VLV_TX_DW2(ch) _PORT(ch, _VLV_TX_DW2_CH0, _VLV_TX_DW2_CH1) #define _VLV_TX_DW3_CH0 0x828c #define _VLV_TX_DW3_CH1 0x848c +/* The following bit for CHV phy */ +#define DPIO_TX_UNIQ_TRANS_SCALE_EN (1<<27) #define VLV_TX_DW3(ch) _PORT(ch, _VLV_TX_DW3_CH0, _VLV_TX_DW3_CH1) #define _VLV_TX_DW4_CH0 0x8290 #define _VLV_TX_DW4_CH1 0x8490 +#define DPIO_SWING_DEEMPH9P5_SHIFT 24 +#define DPIO_SWING_DEEMPH9P5_MASK (0xff << DPIO_SWING_DEEMPH9P5_SHIFT) #define VLV_TX_DW4(ch) _PORT(ch, _VLV_TX_DW4_CH0, _VLV_TX_DW4_CH1) #define _VLV_TX3_DW4_CH0 0x690 @@ -595,6 +839,73 @@ #define _VLV_TX_DW14_CH1 0x84b8 #define VLV_TX_DW14(ch) _PORT(ch, _VLV_TX_DW14_CH0, _VLV_TX_DW14_CH1) +/* CHV dpPhy registers */ +#define _CHV_PLL_DW0_CH0 0x8000 +#define _CHV_PLL_DW0_CH1 0x8180 +#define CHV_PLL_DW0(ch) _PIPE(ch, _CHV_PLL_DW0_CH0, _CHV_PLL_DW0_CH1) + +#define _CHV_PLL_DW1_CH0 0x8004 +#define _CHV_PLL_DW1_CH1 0x8184 +#define DPIO_CHV_N_DIV_SHIFT 8 +#define DPIO_CHV_M1_DIV_BY_2 (0 << 0) +#define CHV_PLL_DW1(ch) _PIPE(ch, _CHV_PLL_DW1_CH0, _CHV_PLL_DW1_CH1) + +#define _CHV_PLL_DW2_CH0 0x8008 +#define _CHV_PLL_DW2_CH1 0x8188 +#define CHV_PLL_DW2(ch) _PIPE(ch, _CHV_PLL_DW2_CH0, _CHV_PLL_DW2_CH1) + +#define _CHV_PLL_DW3_CH0 0x800c +#define _CHV_PLL_DW3_CH1 0x818c +#define DPIO_CHV_FRAC_DIV_EN (1 << 16) +#define DPIO_CHV_FIRST_MOD (0 << 8) +#define DPIO_CHV_SECOND_MOD (1 << 8) +#define DPIO_CHV_FEEDFWD_GAIN_SHIFT 0 +#define CHV_PLL_DW3(ch) _PIPE(ch, _CHV_PLL_DW3_CH0, _CHV_PLL_DW3_CH1) + +#define _CHV_PLL_DW6_CH0 0x8018 +#define _CHV_PLL_DW6_CH1 0x8198 +#define DPIO_CHV_GAIN_CTRL_SHIFT 16 +#define DPIO_CHV_INT_COEFF_SHIFT 8 +#define DPIO_CHV_PROP_COEFF_SHIFT 0 +#define CHV_PLL_DW6(ch) _PIPE(ch, _CHV_PLL_DW6_CH0, _CHV_PLL_DW6_CH1) + +#define _CHV_CMN_DW13_CH0 0x8134 +#define _CHV_CMN_DW0_CH1 0x8080 +#define DPIO_CHV_S1_DIV_SHIFT 21 +#define DPIO_CHV_P1_DIV_SHIFT 13 /* 3 bits */ +#define DPIO_CHV_P2_DIV_SHIFT 8 /* 5 bits */ +#define DPIO_CHV_K_DIV_SHIFT 4 +#define DPIO_PLL_FREQLOCK (1 << 1) +#define DPIO_PLL_LOCK (1 << 0) +#define CHV_CMN_DW13(ch) _PIPE(ch, _CHV_CMN_DW13_CH0, _CHV_CMN_DW0_CH1) + +#define _CHV_CMN_DW14_CH0 0x8138 +#define _CHV_CMN_DW1_CH1 0x8084 +#define DPIO_AFC_RECAL (1 << 14) +#define DPIO_DCLKP_EN (1 << 13) +#define CHV_CMN_DW14(ch) _PIPE(ch, _CHV_CMN_DW14_CH0, _CHV_CMN_DW1_CH1) + +#define CHV_CMN_DW30 0x8178 +#define DPIO_LRC_BYPASS (1 << 3) + +#define _TXLANE(ch, lane, offset) ((ch ? 0x2400 : 0) + \ + (lane) * 0x200 + (offset)) + +#define CHV_TX_DW0(ch, lane) _TXLANE(ch, lane, 0x80) +#define CHV_TX_DW1(ch, lane) _TXLANE(ch, lane, 0x84) +#define CHV_TX_DW2(ch, lane) _TXLANE(ch, lane, 0x88) +#define CHV_TX_DW3(ch, lane) _TXLANE(ch, lane, 0x8c) +#define CHV_TX_DW4(ch, lane) _TXLANE(ch, lane, 0x90) +#define CHV_TX_DW5(ch, lane) _TXLANE(ch, lane, 0x94) +#define CHV_TX_DW6(ch, lane) _TXLANE(ch, lane, 0x98) +#define CHV_TX_DW7(ch, lane) _TXLANE(ch, lane, 0x9c) +#define CHV_TX_DW8(ch, lane) _TXLANE(ch, lane, 0xa0) +#define CHV_TX_DW9(ch, lane) _TXLANE(ch, lane, 0xa4) +#define CHV_TX_DW10(ch, lane) _TXLANE(ch, lane, 0xa8) +#define CHV_TX_DW11(ch, lane) _TXLANE(ch, lane, 0xac) +#define DPIO_FRC_LATENCY_SHFIT 8 +#define CHV_TX_DW14(ch, lane) _TXLANE(ch, lane, 0xb8) +#define DPIO_UPAR_SHIFT 30 /* * Fence registers */ @@ -631,10 +942,14 @@ /* * Instruction and interrupt control regs */ +#define PGTBL_CTL 0x02020 +#define PGTBL_ADDRESS_LO_MASK 0xfffff000 /* bits [31:12] */ +#define PGTBL_ADDRESS_HI_MASK 0x000000f0 /* bits [35:32] (gen4) */ #define PGTBL_ER 0x02024 #define RENDER_RING_BASE 0x02000 #define BSD_RING_BASE 0x04000 #define GEN6_BSD_RING_BASE 0x12000 +#define GEN8_BSD2_RING_BASE 0x1c000 #define VEBOX_RING_BASE 0x1a000 #define BLT_RING_BASE 0x22000 #define RING_TAIL(base) ((base)+0x30) @@ -660,9 +975,20 @@ #define RING_MAX_IDLE(base) ((base)+0x54) #define RING_HWS_PGA(base) ((base)+0x80) #define RING_HWS_PGA_GEN6(base) ((base)+0x2080) -#define ARB_MODE 0x04030 + +#define GEN7_WR_WATERMARK 0x4028 +#define GEN7_GFX_PRIO_CTRL 0x402C +#define ARB_MODE 0x4030 #define ARB_MODE_SWIZZLE_SNB (1<<4) #define ARB_MODE_SWIZZLE_IVB (1<<5) +#define GEN7_GFX_PEND_TLB0 0x4034 +#define GEN7_GFX_PEND_TLB1 0x4038 +/* L3, CVS, ZTLB, RCC, CASC LRA min, max values */ +#define GEN7_LRA_LIMITS_BASE 0x403C +#define GEN7_LRA_LIMITS_REG_NUM 13 +#define GEN7_MEDIA_MAX_REQ_COUNT 0x4070 +#define GEN7_GFX_MAX_REQ_COUNT 0x4074 + #define GAMTARBMODE 0x04a08 #define ARB_MODE_BWGTLB_DISABLE (1<<9) #define ARB_MODE_SWIZZLE_BDW (1<<1) @@ -678,6 +1004,7 @@ #define BLT_HWS_PGA_GEN7 (0x04280) #define VEBOX_HWS_PGA_GEN7 (0x04380) #define RING_ACTHD(base) ((base)+0x74) +#define RING_ACTHD_UDW(base) ((base)+0x5c) #define RING_NOPID(base) ((base)+0x94) #define RING_IMR(base) ((base)+0xa8) #define RING_TIMESTAMP(base) ((base)+0x358) @@ -696,6 +1023,9 @@ #define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */ #define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */ #define RING_WAIT_SEMAPHORE (1<<10) /* gen6+ */ + +#define GEN7_TLB_RD_ADDR 0x4700 + #if 0 #define PRB0_TAIL 0x02030 #define PRB0_HEAD 0x02034 @@ -719,7 +1049,9 @@ #define RING_INSTDONE(base) ((base)+0x6c) #define RING_INSTPS(base) ((base)+0x70) #define RING_DMA_FADD(base) ((base)+0x78) +#define RING_DMA_FADD_UDW(base) ((base)+0x60) /* gen8+ */ #define RING_INSTPM(base) ((base)+0xc0) +#define RING_MI_MODE(base) ((base)+0x9c) #define INSTPS 0x02070 /* 965+ only */ #define INSTDONE1 0x0207c /* 965+ only */ #define ACTHD_I965 0x02074 @@ -789,36 +1121,49 @@ #define _3D_CHICKEN3 0x02090 #define _3D_CHICKEN_SF_DISABLE_OBJEND_CULL (1 << 10) #define _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL (1 << 5) -#define _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x) ((x)<<1) +#define _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x) ((x)<<1) /* gen8+ */ +#define _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH (1 << 1) /* gen6 */ #define MI_MODE 0x0209c # define VS_TIMER_DISPATCH (1 << 6) # define MI_FLUSH_ENABLE (1 << 12) # define ASYNC_FLIP_PERF_DISABLE (1 << 14) +# define MODE_IDLE (1 << 9) +# define STOP_RING (1 << 8) #define GEN6_GT_MODE 0x20d0 -#define GEN6_GT_MODE_HI (1 << 9) +#define GEN7_GT_MODE 0x7008 +#define GEN6_WIZ_HASHING(hi, lo) (((hi) << 9) | ((lo) << 7)) +#define GEN6_WIZ_HASHING_8x8 GEN6_WIZ_HASHING(0, 0) +#define GEN6_WIZ_HASHING_8x4 GEN6_WIZ_HASHING(0, 1) +#define GEN6_WIZ_HASHING_16x4 GEN6_WIZ_HASHING(1, 0) +#define GEN6_WIZ_HASHING_MASK (GEN6_WIZ_HASHING(1, 1) << 16) #define GEN6_TD_FOUR_ROW_DISPATCH_DISABLE (1 << 5) #define GFX_MODE 0x02520 #define GFX_MODE_GEN7 0x0229c #define RING_MODE_GEN7(ring) ((ring)->mmio_base+0x29c) #define GFX_RUN_LIST_ENABLE (1<<15) -#define GFX_TLB_INVALIDATE_ALWAYS (1<<13) +#define GFX_TLB_INVALIDATE_EXPLICIT (1<<13) #define GFX_SURFACE_FAULT_ENABLE (1<<12) #define GFX_REPLAY_MODE (1<<11) #define GFX_PSMI_GRANULARITY (1<<10) #define GFX_PPGTT_ENABLE (1<<9) #define VLV_DISPLAY_BASE 0x180000 +#define VLV_MIPI_BASE VLV_DISPLAY_BASE +#define VLV_GU_CTL0 (VLV_DISPLAY_BASE + 0x2030) +#define VLV_GU_CTL1 (VLV_DISPLAY_BASE + 0x2034) #define SCPD0 0x0209c /* 915+ only */ #define IER 0x020a0 #define IIR 0x020a4 #define IMR 0x020a8 #define ISR 0x020ac #define VLV_GUNIT_CLOCK_GATE (VLV_DISPLAY_BASE + 0x2060) +#define GINT_DIS (1<<22) #define GCFG_DIS (1<<8) +#define VLV_GUNIT_CLOCK_GATE2 (VLV_DISPLAY_BASE + 0x2064) #define VLV_IIR_RW (VLV_DISPLAY_BASE + 0x2084) #define VLV_IER (VLV_DISPLAY_BASE + 0x20a0) #define VLV_IIR (VLV_DISPLAY_BASE + 0x20a4) @@ -837,7 +1182,7 @@ #define I915_ERROR_INSTRUCTION (1<<0) #define INSTPM 0x020c0 #define INSTPM_SELF_EN (1<<12) /* 915GM only */ -#define INSTPM_AGPBUSY_DIS (1<<11) /* gen3: when disabled, pending interrupts +#define INSTPM_AGPBUSY_INT_EN (1<<11) /* gen3: when disabled, pending interrupts will not assert AGPBUSY# and will only be delivered when out of C3. */ #define INSTPM_FORCE_ORDERING (1<<7) /* GEN6+ */ @@ -918,6 +1263,10 @@ #define MI_ARB_DISPLAY_PRIORITY_A_B (0 << 0) /* display A > display B */ #define MI_ARB_DISPLAY_PRIORITY_B_A (1 << 0) /* display B > display A */ +#define MI_STATE 0x020e4 /* gen2 only */ +#define MI_AGPBUSY_INT_EN (1 << 1) /* 85x only */ +#define MI_AGPBUSY_830_MODE (1 << 0) /* 85x only */ + #define CACHE_MODE_0 0x02120 /* 915+ only */ #define CM0_PIPELINED_RENDER_FLUSH_DISABLE (1<<8) #define CM0_IZ_OPT_DISABLE (1<<6) @@ -934,13 +1283,21 @@ #define ECO_GATING_CX_ONLY (1<<3) #define ECO_FLIP_DONE (1<<0) +#define CACHE_MODE_0_GEN7 0x7000 /* IVB+ */ +#define RC_OP_FLUSH_ENABLE (1<<0) +#define HIZ_RAW_STALL_OPT_DISABLE (1<<2) #define CACHE_MODE_1 0x7004 /* IVB+ */ -#define PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1<<6) +#define PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1<<6) +#define GEN8_4x4_STC_OPTIMIZATION_DISABLE (1<<6) #define GEN6_BLITTER_ECOSKPD 0x221d0 #define GEN6_BLITTER_LOCK_SHIFT 16 #define GEN6_BLITTER_FBC_NOTIFY (1<<3) +#define GEN6_RC_SLEEP_PSMI_CONTROL 0x2050 +#define GEN8_RC_SEMA_IDLE_MSG_DISABLE (1 << 12) +#define GEN8_FF_DOP_CLOCK_GATE_DISABLE (1<<10) + #define GEN6_BSD_SLEEP_PSMI_CONTROL 0x12050 #define GEN6_BSD_SLEEP_MSG_DISABLE (1 << 0) #define GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2) @@ -980,24 +1337,43 @@ /* These are all the "old" interrupts */ #define ILK_BSD_USER_INTERRUPT (1<<5) + +#define I915_PM_INTERRUPT (1<<31) +#define I915_ISP_INTERRUPT (1<<22) +#define I915_LPE_PIPE_B_INTERRUPT (1<<21) +#define I915_LPE_PIPE_A_INTERRUPT (1<<20) +#define I915_MIPIB_INTERRUPT (1<<19) +#define I915_MIPIA_INTERRUPT (1<<18) #define I915_PIPE_CONTROL_NOTIFY_INTERRUPT (1<<18) #define I915_DISPLAY_PORT_INTERRUPT (1<<17) +#define I915_DISPLAY_PIPE_C_HBLANK_INTERRUPT (1<<16) +#define I915_MASTER_ERROR_INTERRUPT (1<<15) #define I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT (1<<15) +#define I915_DISPLAY_PIPE_B_HBLANK_INTERRUPT (1<<14) #define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT (1<<14) /* p-state */ +#define I915_DISPLAY_PIPE_A_HBLANK_INTERRUPT (1<<13) #define I915_HWB_OOM_INTERRUPT (1<<13) +#define I915_LPE_PIPE_C_INTERRUPT (1<<12) #define I915_SYNC_STATUS_INTERRUPT (1<<12) +#define I915_MISC_INTERRUPT (1<<11) #define I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT (1<<11) +#define I915_DISPLAY_PIPE_C_VBLANK_INTERRUPT (1<<10) #define I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT (1<<10) +#define I915_DISPLAY_PIPE_C_EVENT_INTERRUPT (1<<9) #define I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT (1<<9) +#define I915_DISPLAY_PIPE_C_DPBM_INTERRUPT (1<<8) #define I915_DISPLAY_PLANE_C_FLIP_PENDING_INTERRUPT (1<<8) #define I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT (1<<7) #define I915_DISPLAY_PIPE_A_EVENT_INTERRUPT (1<<6) #define I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT (1<<5) #define I915_DISPLAY_PIPE_B_EVENT_INTERRUPT (1<<4) +#define I915_DISPLAY_PIPE_A_DPBM_INTERRUPT (1<<3) +#define I915_DISPLAY_PIPE_B_DPBM_INTERRUPT (1<<2) #define I915_DEBUG_INTERRUPT (1<<2) +#define I915_WINVALID_INTERRUPT (1<<1) #define I915_USER_INTERRUPT (1<<1) #define I915_ASLE_INTERRUPT (1<<0) -#define I915_BSD_USER_INTERRUPT (1 << 25) +#define I915_BSD_USER_INTERRUPT (1<<25) #define GEN6_BSD_RNCID 0x12198 @@ -1046,9 +1422,8 @@ #define FBC_CTL_IDLE_LINE (2<<2) #define FBC_CTL_IDLE_DEBUG (3<<2) #define FBC_CTL_CPU_FENCE (1<<1) -#define FBC_CTL_PLANEA (0<<0) -#define FBC_CTL_PLANEB (1<<0) -#define FBC_FENCE_OFF 0x0321b +#define FBC_CTL_PLANE(plane) ((plane)<<0) +#define FBC_FENCE_OFF 0x03218 /* BSpec typo has 321Bh */ #define FBC_TAG 0x03300 #define FBC_LL_SIZE (1536) @@ -1057,9 +1432,8 @@ #define DPFC_CB_BASE 0x3200 #define DPFC_CONTROL 0x3208 #define DPFC_CTL_EN (1<<31) -#define DPFC_CTL_PLANEA (0<<30) -#define DPFC_CTL_PLANEB (1<<30) -#define IVB_DPFC_CTL_PLANE_SHIFT (29) +#define DPFC_CTL_PLANE(plane) ((plane)<<30) +#define IVB_DPFC_CTL_PLANE(plane) ((plane)<<29) #define DPFC_CTL_FENCE_EN (1<<29) #define IVB_DPFC_CTL_FENCE_EN (1<<28) #define DPFC_CTL_PERSISTENT_MODE (1<<25) @@ -1120,13 +1494,6 @@ #define FBC_REND_NUKE (1<<2) #define FBC_REND_CACHE_CLEAN (1<<1) -#define _HSW_PIPE_SLICE_CHICKEN_1_A 0x420B0 -#define _HSW_PIPE_SLICE_CHICKEN_1_B 0x420B4 -#define HSW_BYPASS_FBC_QUEUE (1<<22) -#define HSW_PIPE_SLICE_CHICKEN_1(pipe) _PIPE(pipe, + \ - _HSW_PIPE_SLICE_CHICKEN_1_A, + \ - _HSW_PIPE_SLICE_CHICKEN_1_B) - /* * GPIO regs */ @@ -1163,6 +1530,7 @@ #define GMBUS_PORT_SSC 1 #define GMBUS_PORT_VGADDC 2 #define GMBUS_PORT_PANEL 3 +#define GMBUS_PORT_DPD_CHV 3 /* HDMID_CHV */ #define GMBUS_PORT_DPC 4 /* HDMIC */ #define GMBUS_PORT_DPB 5 /* SDVO, HDMIB */ #define GMBUS_PORT_DPD 6 /* HDMID */ @@ -1202,6 +1570,11 @@ /* * Clock control & power management */ +#define DPLL_A_OFFSET 0x6014 +#define DPLL_B_OFFSET 0x6018 +#define CHV_DPLL_C_OFFSET 0x6030 +#define DPLL(pipe) (dev_priv->info.dpll_offsets[pipe] + \ + dev_priv->info.display_mmio_offset) #define VGA0 0x6000 #define VGA1 0x6004 @@ -1214,9 +1587,6 @@ #define VGA1_PD_P1_DIV_2 (1 << 13) #define VGA1_PD_P1_SHIFT 8 #define VGA1_PD_P1_MASK (0x1f << 8) -#define _DPLL_A (dev_priv->info->display_mmio_offset + 0x6014) -#define _DPLL_B (dev_priv->info->display_mmio_offset + 0x6018) -#define DPLL(pipe) _PIPE(pipe, _DPLL_A, _DPLL_B) #define DPLL_VCO_ENABLE (1 << 31) #define DPLL_SDVO_HIGH_SPEED (1 << 30) #define DPLL_DVO_2X_MODE (1 << 30) @@ -1237,10 +1607,23 @@ #define DPLL_LOCK_VLV (1<<15) #define DPLL_INTEGRATED_CRI_CLK_VLV (1<<14) #define DPLL_INTEGRATED_CLOCK_VLV (1<<13) +#define DPLL_SSC_REF_CLOCK_CHV (1<<13) #define DPLL_PORTC_READY_MASK (0xf << 4) #define DPLL_PORTB_READY_MASK (0xf) #define DPLL_FPA01_P1_POST_DIV_MASK_I830 0x001f0000 + +/* Additional CHV pll/phy registers */ +#define DPIO_PHY_STATUS (VLV_DISPLAY_BASE + 0x6240) +#define DPLL_PORTD_READY_MASK (0xf) +#define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100) +#define PHY_COM_LANE_RESET_DEASSERT(phy, val) \ + ((phy == DPIO_PHY0) ? (val | 1) : (val | 2)) +#define PHY_COM_LANE_RESET_ASSERT(phy, val) \ + ((phy == DPIO_PHY0) ? (val & ~1) : (val & ~2)) +#define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104) +#define PHY_POWERGOOD(phy) ((phy == DPIO_PHY0) ? (1<<31) : (1<<30)) + /* * The i830 generation, in LVDS mode, defines P1 as the bit number set within * this field (only one bit may be set). @@ -1278,7 +1661,13 @@ #define SDVO_MULTIPLIER_MASK 0x000000ff #define SDVO_MULTIPLIER_SHIFT_HIRES 4 #define SDVO_MULTIPLIER_SHIFT_VGA 0 -#define _DPLL_A_MD (dev_priv->info->display_mmio_offset + 0x601c) /* 965+ only */ + +#define DPLL_A_MD_OFFSET 0x601c /* 965+ only */ +#define DPLL_B_MD_OFFSET 0x6020 /* 965+ only */ +#define CHV_DPLL_C_MD_OFFSET 0x603c +#define DPLL_MD(pipe) (dev_priv->info.dpll_md_offsets[pipe] + \ + dev_priv->info.display_mmio_offset) + /* * UDI pixel divider, controlling how many pixels are stuffed into a packet. * @@ -1315,8 +1704,6 @@ */ #define DPLL_MD_VGA_UDI_MULTIPLIER_MASK 0x0000003f #define DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT 0 -#define _DPLL_B_MD (dev_priv->info->display_mmio_offset + 0x6020) /* 965+ only */ -#define DPLL_MD(pipe) _PIPE(pipe, _DPLL_A_MD, _DPLL_B_MD) #define _FPA0 0x06040 #define _FPA1 0x06044 @@ -1348,7 +1735,7 @@ #define DSTATE_PLL_D3_OFF (1<<3) #define DSTATE_GFX_CLOCK_GATING (1<<1) #define DSTATE_DOT_CLOCK_GATING (1<<0) -#define DSPCLK_GATE_D (dev_priv->info->display_mmio_offset + 0x6200) +#define DSPCLK_GATE_D (dev_priv->info.display_mmio_offset + 0x6200) # define DPUNIT_B_CLOCK_GATE_DISABLE (1 << 30) /* 965 */ # define VSUNIT_CLOCK_GATE_DISABLE (1 << 29) /* 965 */ # define VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) /* 965 */ @@ -1377,7 +1764,7 @@ # define DPIOUNIT_CLOCK_GATE_DISABLE (1 << 6) /* 915-945 */ # define OVFUNIT_CLOCK_GATE_DISABLE (1 << 5) # define OVBUNIT_CLOCK_GATE_DISABLE (1 << 4) -/** +/* * This bit must be set on the 830 to prevent hangs when turning off the * overlay scaler. */ @@ -1397,12 +1784,12 @@ # define COLOR_CALCULATOR_CLOCK_GATE_DISABLE (1 << 7) # define MOTION_COMP_CLOCK_GATE_DISABLE (1 << 6) # define MAG_CLOCK_GATE_DISABLE (1 << 5) -/** This bit must be unset on 855,865 */ +/* This bit must be unset on 855,865 */ # define MECI_CLOCK_GATE_DISABLE (1 << 4) # define DCMP_CLOCK_GATE_DISABLE (1 << 3) # define MEC_CLOCK_GATE_DISABLE (1 << 2) # define MECO_CLOCK_GATE_DISABLE (1 << 1) -/** This bit must be set on 855,865. */ +/* This bit must be set on 855,865. */ # define SV_CLOCK_GATE_DISABLE (1 << 0) # define I915_MPEG_CLOCK_GATE_DISABLE (1 << 16) # define I915_VLD_IP_PR_CLOCK_GATE_DISABLE (1 << 15) @@ -1423,14 +1810,14 @@ # define I915_BY_CLOCK_GATE_DISABLE (1 << 0) # define I965_RCZ_CLOCK_GATE_DISABLE (1 << 30) -/** This bit must always be set on 965G/965GM */ +/* This bit must always be set on 965G/965GM */ # define I965_RCC_CLOCK_GATE_DISABLE (1 << 29) # define I965_RCPB_CLOCK_GATE_DISABLE (1 << 28) # define I965_DAP_CLOCK_GATE_DISABLE (1 << 27) # define I965_ROC_CLOCK_GATE_DISABLE (1 << 26) # define I965_GW_CLOCK_GATE_DISABLE (1 << 25) # define I965_TD_CLOCK_GATE_DISABLE (1 << 24) -/** This bit must always be set on 965G */ +/* This bit must always be set on 965G */ # define I965_ISC_CLOCK_GATE_DISABLE (1 << 23) # define I965_IC_CLOCK_GATE_DISABLE (1 << 22) # define I965_EU_CLOCK_GATE_DISABLE (1 << 21) @@ -1455,6 +1842,10 @@ #define VF_UNIT_CLOCK_GATE_DISABLE (1 << 9) #define GS_UNIT_CLOCK_GATE_DISABLE (1 << 7) #define CL_UNIT_CLOCK_GATE_DISABLE (1 << 6) + +#define VDECCLK_GATE_D 0x620C /* g4x only */ +#define VCP_UNIT_CLOCK_GATE_DISABLE (1 << 4) + #define RAMCLK_GATE_D 0x6210 /* CRL only */ #define DEUC 0x6214 /* CRL only */ @@ -1472,10 +1863,11 @@ /* * Palette regs */ - -#define _PALETTE_A (dev_priv->info->display_mmio_offset + 0xa000) -#define _PALETTE_B (dev_priv->info->display_mmio_offset + 0xa800) -#define PALETTE(pipe) _PIPE(pipe, _PALETTE_A, _PALETTE_B) +#define PALETTE_A_OFFSET 0xa000 +#define PALETTE_B_OFFSET 0xa800 +#define CHV_PALETTE_C_OFFSET 0xc000 +#define PALETTE(pipe) (dev_priv->info.palette_offsets[pipe] + \ + dev_priv->info.display_mmio_offset) /* MCH MMIO space */ @@ -1496,7 +1888,7 @@ /* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */ #define DCLK (MCHBAR_MIRROR_BASE_SNB + 0x5e04) -/** 915-945 and GM965 MCH register controlling DRAM channel access */ +/* 915-945 and GM965 MCH register controlling DRAM channel access */ #define DCC 0x10200 #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0) @@ -1505,15 +1897,15 @@ #define DCC_CHANNEL_XOR_DISABLE (1 << 10) #define DCC_CHANNEL_XOR_BIT_17 (1 << 9) -/** Pineview MCH register contains DDR3 setting */ +/* Pineview MCH register contains DDR3 setting */ #define CSHRDDR3CTL 0x101a8 #define CSHRDDR3CTL_DDR3 (1 << 2) -/** 965 MCH register controlling DRAM channel configuration */ +/* 965 MCH register controlling DRAM channel configuration */ #define C0DRB3 0x10206 #define C1DRB3 0x10606 -/** snb MCH registers for reading the DRAM channel configuration */ +/* snb MCH registers for reading the DRAM channel configuration */ #define MAD_DIMM_C0 (MCHBAR_MIRROR_BASE_SNB + 0x5004) #define MAD_DIMM_C1 (MCHBAR_MIRROR_BASE_SNB + 0x5008) #define MAD_DIMM_C2 (MCHBAR_MIRROR_BASE_SNB + 0x500C) @@ -1535,7 +1927,7 @@ #define MAD_DIMM_A_SIZE_SHIFT 0 #define MAD_DIMM_A_SIZE_MASK (0xff << MAD_DIMM_A_SIZE_SHIFT) -/** snb MCH registers for priority tuning */ +/* snb MCH registers for priority tuning */ #define MCH_SSKPD (MCHBAR_MIRROR_BASE_SNB + 0x5d10) #define MCH_SSKPD_WM0_MASK 0x3f #define MCH_SSKPD_WM0_VAL 0xc @@ -1862,7 +2254,7 @@ */ /* Pipe A CRC regs */ -#define _PIPE_CRC_CTL_A (dev_priv->info->display_mmio_offset + 0x60050) +#define _PIPE_CRC_CTL_A 0x60050 #define PIPE_CRC_ENABLE (1 << 31) /* ivb+ source selection */ #define PIPE_CRC_SOURCE_PRIMARY_IVB (0 << 29) @@ -1902,11 +2294,11 @@ #define _PIPE_CRC_RES_4_A_IVB 0x60070 #define _PIPE_CRC_RES_5_A_IVB 0x60074 -#define _PIPE_CRC_RES_RED_A (dev_priv->info->display_mmio_offset + 0x60060) -#define _PIPE_CRC_RES_GREEN_A (dev_priv->info->display_mmio_offset + 0x60064) -#define _PIPE_CRC_RES_BLUE_A (dev_priv->info->display_mmio_offset + 0x60068) -#define _PIPE_CRC_RES_RES1_A_I915 (dev_priv->info->display_mmio_offset + 0x6006c) -#define _PIPE_CRC_RES_RES2_A_G4X (dev_priv->info->display_mmio_offset + 0x60080) +#define _PIPE_CRC_RES_RED_A 0x60060 +#define _PIPE_CRC_RES_GREEN_A 0x60064 +#define _PIPE_CRC_RES_BLUE_A 0x60068 +#define _PIPE_CRC_RES_RES1_A_I915 0x6006c +#define _PIPE_CRC_RES_RES2_A_G4X 0x60080 /* Pipe B CRC regs */ #define _PIPE_CRC_RES_1_B_IVB 0x61064 @@ -1915,59 +2307,70 @@ #define _PIPE_CRC_RES_4_B_IVB 0x61070 #define _PIPE_CRC_RES_5_B_IVB 0x61074 -#define PIPE_CRC_CTL(pipe) _PIPE_INC(pipe, _PIPE_CRC_CTL_A, 0x01000) +#define PIPE_CRC_CTL(pipe) _TRANSCODER2(pipe, _PIPE_CRC_CTL_A) #define PIPE_CRC_RES_1_IVB(pipe) \ - _PIPE(pipe, _PIPE_CRC_RES_1_A_IVB, _PIPE_CRC_RES_1_B_IVB) + _TRANSCODER2(pipe, _PIPE_CRC_RES_1_A_IVB) #define PIPE_CRC_RES_2_IVB(pipe) \ - _PIPE(pipe, _PIPE_CRC_RES_2_A_IVB, _PIPE_CRC_RES_2_B_IVB) + _TRANSCODER2(pipe, _PIPE_CRC_RES_2_A_IVB) #define PIPE_CRC_RES_3_IVB(pipe) \ - _PIPE(pipe, _PIPE_CRC_RES_3_A_IVB, _PIPE_CRC_RES_3_B_IVB) + _TRANSCODER2(pipe, _PIPE_CRC_RES_3_A_IVB) #define PIPE_CRC_RES_4_IVB(pipe) \ - _PIPE(pipe, _PIPE_CRC_RES_4_A_IVB, _PIPE_CRC_RES_4_B_IVB) + _TRANSCODER2(pipe, _PIPE_CRC_RES_4_A_IVB) #define PIPE_CRC_RES_5_IVB(pipe) \ - _PIPE(pipe, _PIPE_CRC_RES_5_A_IVB, _PIPE_CRC_RES_5_B_IVB) + _TRANSCODER2(pipe, _PIPE_CRC_RES_5_A_IVB) #define PIPE_CRC_RES_RED(pipe) \ - _PIPE_INC(pipe, _PIPE_CRC_RES_RED_A, 0x01000) + _TRANSCODER2(pipe, _PIPE_CRC_RES_RED_A) #define PIPE_CRC_RES_GREEN(pipe) \ - _PIPE_INC(pipe, _PIPE_CRC_RES_GREEN_A, 0x01000) + _TRANSCODER2(pipe, _PIPE_CRC_RES_GREEN_A) #define PIPE_CRC_RES_BLUE(pipe) \ - _PIPE_INC(pipe, _PIPE_CRC_RES_BLUE_A, 0x01000) + _TRANSCODER2(pipe, _PIPE_CRC_RES_BLUE_A) #define PIPE_CRC_RES_RES1_I915(pipe) \ - _PIPE_INC(pipe, _PIPE_CRC_RES_RES1_A_I915, 0x01000) + _TRANSCODER2(pipe, _PIPE_CRC_RES_RES1_A_I915) #define PIPE_CRC_RES_RES2_G4X(pipe) \ - _PIPE_INC(pipe, _PIPE_CRC_RES_RES2_A_G4X, 0x01000) + _TRANSCODER2(pipe, _PIPE_CRC_RES_RES2_A_G4X) /* Pipe A timing regs */ -#define _HTOTAL_A (dev_priv->info->display_mmio_offset + 0x60000) -#define _HBLANK_A (dev_priv->info->display_mmio_offset + 0x60004) -#define _HSYNC_A (dev_priv->info->display_mmio_offset + 0x60008) -#define _VTOTAL_A (dev_priv->info->display_mmio_offset + 0x6000c) -#define _VBLANK_A (dev_priv->info->display_mmio_offset + 0x60010) -#define _VSYNC_A (dev_priv->info->display_mmio_offset + 0x60014) -#define _PIPEASRC (dev_priv->info->display_mmio_offset + 0x6001c) -#define _BCLRPAT_A (dev_priv->info->display_mmio_offset + 0x60020) -#define _VSYNCSHIFT_A (dev_priv->info->display_mmio_offset + 0x60028) +#define _HTOTAL_A 0x60000 +#define _HBLANK_A 0x60004 +#define _HSYNC_A 0x60008 +#define _VTOTAL_A 0x6000c +#define _VBLANK_A 0x60010 +#define _VSYNC_A 0x60014 +#define _PIPEASRC 0x6001c +#define _BCLRPAT_A 0x60020 +#define _VSYNCSHIFT_A 0x60028 /* Pipe B timing regs */ -#define _HTOTAL_B (dev_priv->info->display_mmio_offset + 0x61000) -#define _HBLANK_B (dev_priv->info->display_mmio_offset + 0x61004) -#define _HSYNC_B (dev_priv->info->display_mmio_offset + 0x61008) -#define _VTOTAL_B (dev_priv->info->display_mmio_offset + 0x6100c) -#define _VBLANK_B (dev_priv->info->display_mmio_offset + 0x61010) -#define _VSYNC_B (dev_priv->info->display_mmio_offset + 0x61014) -#define _PIPEBSRC (dev_priv->info->display_mmio_offset + 0x6101c) -#define _BCLRPAT_B (dev_priv->info->display_mmio_offset + 0x61020) -#define _VSYNCSHIFT_B (dev_priv->info->display_mmio_offset + 0x61028) - -#define HTOTAL(trans) _TRANSCODER(trans, _HTOTAL_A, _HTOTAL_B) -#define HBLANK(trans) _TRANSCODER(trans, _HBLANK_A, _HBLANK_B) -#define HSYNC(trans) _TRANSCODER(trans, _HSYNC_A, _HSYNC_B) -#define VTOTAL(trans) _TRANSCODER(trans, _VTOTAL_A, _VTOTAL_B) -#define VBLANK(trans) _TRANSCODER(trans, _VBLANK_A, _VBLANK_B) -#define VSYNC(trans) _TRANSCODER(trans, _VSYNC_A, _VSYNC_B) -#define BCLRPAT(pipe) _PIPE(pipe, _BCLRPAT_A, _BCLRPAT_B) -#define VSYNCSHIFT(trans) _TRANSCODER(trans, _VSYNCSHIFT_A, _VSYNCSHIFT_B) +#define _HTOTAL_B 0x61000 +#define _HBLANK_B 0x61004 +#define _HSYNC_B 0x61008 +#define _VTOTAL_B 0x6100c +#define _VBLANK_B 0x61010 +#define _VSYNC_B 0x61014 +#define _PIPEBSRC 0x6101c +#define _BCLRPAT_B 0x61020 +#define _VSYNCSHIFT_B 0x61028 + +#define TRANSCODER_A_OFFSET 0x60000 +#define TRANSCODER_B_OFFSET 0x61000 +#define TRANSCODER_C_OFFSET 0x62000 +#define CHV_TRANSCODER_C_OFFSET 0x63000 +#define TRANSCODER_EDP_OFFSET 0x6f000 + +#define _TRANSCODER2(pipe, reg) (dev_priv->info.trans_offsets[(pipe)] - \ + dev_priv->info.trans_offsets[TRANSCODER_A] + (reg) + \ + dev_priv->info.display_mmio_offset) + +#define HTOTAL(trans) _TRANSCODER2(trans, _HTOTAL_A) +#define HBLANK(trans) _TRANSCODER2(trans, _HBLANK_A) +#define HSYNC(trans) _TRANSCODER2(trans, _HSYNC_A) +#define VTOTAL(trans) _TRANSCODER2(trans, _VTOTAL_A) +#define VBLANK(trans) _TRANSCODER2(trans, _VBLANK_A) +#define VSYNC(trans) _TRANSCODER2(trans, _VSYNC_A) +#define BCLRPAT(trans) _TRANSCODER2(trans, _BCLRPAT_A) +#define VSYNCSHIFT(trans) _TRANSCODER2(trans, _VSYNCSHIFT_A) +#define PIPESRC(trans) _TRANSCODER2(trans, _PIPEASRC) /* HSW+ eDP PSR registers */ #define EDP_PSR_BASE(dev) (IS_HASWELL(dev) ? 0x64800 : 0x6f800) @@ -2084,7 +2487,7 @@ /* Hotplug control (945+ only) */ -#define PORT_HOTPLUG_EN (dev_priv->info->display_mmio_offset + 0x61110) +#define PORT_HOTPLUG_EN (dev_priv->info.display_mmio_offset + 0x61110) #define PORTB_HOTPLUG_INT_EN (1 << 29) #define PORTC_HOTPLUG_INT_EN (1 << 28) #define PORTD_HOTPLUG_INT_EN (1 << 27) @@ -2114,7 +2517,7 @@ #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV (0 << 2) #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV (1 << 2) -#define PORT_HOTPLUG_STAT (dev_priv->info->display_mmio_offset + 0x61114) +#define PORT_HOTPLUG_STAT (dev_priv->info.display_mmio_offset + 0x61114) /* * HDMI/DP bits are gen4+ * @@ -2177,6 +2580,7 @@ #define GEN3_SDVOC 0x61160 #define GEN4_HDMIB GEN3_SDVOB #define GEN4_HDMIC GEN3_SDVOC +#define CHV_HDMID 0x6116C #define PCH_SDVOB 0xe1140 #define PCH_HDMIB PCH_SDVOB #define PCH_HDMIC 0xe1150 @@ -2197,7 +2601,7 @@ #define SDVO_PIPE_B_SELECT (1 << 30) #define SDVO_STALL_SELECT (1 << 29) #define SDVO_INTERRUPT_ENABLE (1 << 26) -/** +/* * 915G/GM SDVO pixel multiplier. * Programmed value is multiplier - 1, up to 5x. * \sa DPLL_MD_UDI_MULTIPLIER_MASK @@ -2237,6 +2641,10 @@ #define SDVO_PIPE_SEL_CPT(pipe) ((pipe) << 29) #define SDVO_PIPE_SEL_MASK_CPT (3 << 29) +/* CHV SDVO/HDMI bits: */ +#define SDVO_PIPE_SEL_CHV(pipe) ((pipe) << 24) +#define SDVO_PIPE_SEL_MASK_CHV (3 << 24) + /* DVO port control */ #define DVOA 0x61120 @@ -2332,9 +2740,7 @@ #define VIDEO_DIP_CTL 0x61170 /* Pre HSW: */ #define VIDEO_DIP_ENABLE (1 << 31) -#define VIDEO_DIP_PORT_B (1 << 29) -#define VIDEO_DIP_PORT_C (2 << 29) -#define VIDEO_DIP_PORT_D (3 << 29) +#define VIDEO_DIP_PORT(port) ((port) << 29) #define VIDEO_DIP_PORT_MASK (3 << 29) #define VIDEO_DIP_ENABLE_GCP (1 << 25) #define VIDEO_DIP_ENABLE_AVI (1 << 21) @@ -2391,7 +2797,7 @@ #define PP_DIVISOR 0x61210 /* Panel fitting */ -#define PFIT_CONTROL (dev_priv->info->display_mmio_offset + 0x61230) +#define PFIT_CONTROL (dev_priv->info.display_mmio_offset + 0x61230) #define PFIT_ENABLE (1 << 31) #define PFIT_PIPE_MASK (3 << 29) #define PFIT_PIPE_SHIFT 29 @@ -2409,7 +2815,7 @@ #define PFIT_SCALING_PROGRAMMED (1 << 26) #define PFIT_SCALING_PILLAR (2 << 26) #define PFIT_SCALING_LETTER (3 << 26) -#define PFIT_PGM_RATIOS (dev_priv->info->display_mmio_offset + 0x61234) +#define PFIT_PGM_RATIOS (dev_priv->info.display_mmio_offset + 0x61234) /* Pre-965 */ #define PFIT_VERT_SCALE_SHIFT 20 #define PFIT_VERT_SCALE_MASK 0xfff00000 @@ -2421,25 +2827,25 @@ #define PFIT_HORIZ_SCALE_SHIFT_965 0 #define PFIT_HORIZ_SCALE_MASK_965 0x00001fff -#define PFIT_AUTO_RATIOS (dev_priv->info->display_mmio_offset + 0x61238) +#define PFIT_AUTO_RATIOS (dev_priv->info.display_mmio_offset + 0x61238) -#define _VLV_BLC_PWM_CTL2_A (dev_priv->info->display_mmio_offset + 0x61250) -#define _VLV_BLC_PWM_CTL2_B (dev_priv->info->display_mmio_offset + 0x61350) +#define _VLV_BLC_PWM_CTL2_A (dev_priv->info.display_mmio_offset + 0x61250) +#define _VLV_BLC_PWM_CTL2_B (dev_priv->info.display_mmio_offset + 0x61350) #define VLV_BLC_PWM_CTL2(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ _VLV_BLC_PWM_CTL2_B) -#define _VLV_BLC_PWM_CTL_A (dev_priv->info->display_mmio_offset + 0x61254) -#define _VLV_BLC_PWM_CTL_B (dev_priv->info->display_mmio_offset + 0x61354) +#define _VLV_BLC_PWM_CTL_A (dev_priv->info.display_mmio_offset + 0x61254) +#define _VLV_BLC_PWM_CTL_B (dev_priv->info.display_mmio_offset + 0x61354) #define VLV_BLC_PWM_CTL(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ _VLV_BLC_PWM_CTL_B) -#define _VLV_BLC_HIST_CTL_A (dev_priv->info->display_mmio_offset + 0x61260) -#define _VLV_BLC_HIST_CTL_B (dev_priv->info->display_mmio_offset + 0x61360) +#define _VLV_BLC_HIST_CTL_A (dev_priv->info.display_mmio_offset + 0x61260) +#define _VLV_BLC_HIST_CTL_B (dev_priv->info.display_mmio_offset + 0x61360) #define VLV_BLC_HIST_CTL(pipe) _PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ _VLV_BLC_HIST_CTL_B) /* Backlight control */ -#define BLC_PWM_CTL2 (dev_priv->info->display_mmio_offset + 0x61250) /* 965+ only */ +#define BLC_PWM_CTL2 (dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */ #define BLM_PWM_ENABLE (1 << 31) #define BLM_COMBINATION_MODE (1 << 30) /* gen4 only */ #define BLM_PIPE_SELECT (1 << 29) @@ -2462,7 +2868,7 @@ #define BLM_PHASE_IN_COUNT_MASK (0xff << 8) #define BLM_PHASE_IN_INCR_SHIFT (0) #define BLM_PHASE_IN_INCR_MASK (0xff << 0) -#define BLC_PWM_CTL (dev_priv->info->display_mmio_offset + 0x61254) +#define BLC_PWM_CTL (dev_priv->info.display_mmio_offset + 0x61254) /* * This is the most significant 15 bits of the number of backlight cycles in a * complete cycle of the modulated backlight control. @@ -2484,7 +2890,7 @@ #define BACKLIGHT_DUTY_CYCLE_MASK_PNV (0xfffe) #define BLM_POLARITY_PNV (1 << 0) /* pnv only */ -#define BLC_HIST_CTL (dev_priv->info->display_mmio_offset + 0x61260) +#define BLC_HIST_CTL (dev_priv->info.display_mmio_offset + 0x61260) /* New registers for PCH-split platforms. Safe where new bits show up, the * register layout machtes with gen4 BLC_PWM_CTL[12]. */ @@ -2509,65 +2915,65 @@ /* TV port control */ #define TV_CTL 0x68000 -/** Enables the TV encoder */ +/* Enables the TV encoder */ # define TV_ENC_ENABLE (1 << 31) -/** Sources the TV encoder input from pipe B instead of A. */ +/* Sources the TV encoder input from pipe B instead of A. */ # define TV_ENC_PIPEB_SELECT (1 << 30) -/** Outputs composite video (DAC A only) */ +/* Outputs composite video (DAC A only) */ # define TV_ENC_OUTPUT_COMPOSITE (0 << 28) -/** Outputs SVideo video (DAC B/C) */ +/* Outputs SVideo video (DAC B/C) */ # define TV_ENC_OUTPUT_SVIDEO (1 << 28) -/** Outputs Component video (DAC A/B/C) */ +/* Outputs Component video (DAC A/B/C) */ # define TV_ENC_OUTPUT_COMPONENT (2 << 28) -/** Outputs Composite and SVideo (DAC A/B/C) */ +/* Outputs Composite and SVideo (DAC A/B/C) */ # define TV_ENC_OUTPUT_SVIDEO_COMPOSITE (3 << 28) # define TV_TRILEVEL_SYNC (1 << 21) -/** Enables slow sync generation (945GM only) */ +/* Enables slow sync generation (945GM only) */ # define TV_SLOW_SYNC (1 << 20) -/** Selects 4x oversampling for 480i and 576p */ +/* Selects 4x oversampling for 480i and 576p */ # define TV_OVERSAMPLE_4X (0 << 18) -/** Selects 2x oversampling for 720p and 1080i */ +/* Selects 2x oversampling for 720p and 1080i */ # define TV_OVERSAMPLE_2X (1 << 18) -/** Selects no oversampling for 1080p */ +/* Selects no oversampling for 1080p */ # define TV_OVERSAMPLE_NONE (2 << 18) -/** Selects 8x oversampling */ +/* Selects 8x oversampling */ # define TV_OVERSAMPLE_8X (3 << 18) -/** Selects progressive mode rather than interlaced */ +/* Selects progressive mode rather than interlaced */ # define TV_PROGRESSIVE (1 << 17) -/** Sets the colorburst to PAL mode. Required for non-M PAL modes. */ +/* Sets the colorburst to PAL mode. Required for non-M PAL modes. */ # define TV_PAL_BURST (1 << 16) -/** Field for setting delay of Y compared to C */ +/* Field for setting delay of Y compared to C */ # define TV_YC_SKEW_MASK (7 << 12) -/** Enables a fix for 480p/576p standard definition modes on the 915GM only */ +/* Enables a fix for 480p/576p standard definition modes on the 915GM only */ # define TV_ENC_SDP_FIX (1 << 11) -/** +/* * Enables a fix for the 915GM only. * * Not sure what it does. */ # define TV_ENC_C0_FIX (1 << 10) -/** Bits that must be preserved by software */ +/* Bits that must be preserved by software */ # define TV_CTL_SAVE ((1 << 11) | (3 << 9) | (7 << 6) | 0xf) # define TV_FUSE_STATE_MASK (3 << 4) -/** Read-only state that reports all features enabled */ +/* Read-only state that reports all features enabled */ # define TV_FUSE_STATE_ENABLED (0 << 4) -/** Read-only state that reports that Macrovision is disabled in hardware*/ +/* Read-only state that reports that Macrovision is disabled in hardware*/ # define TV_FUSE_STATE_NO_MACROVISION (1 << 4) -/** Read-only state that reports that TV-out is disabled in hardware. */ +/* Read-only state that reports that TV-out is disabled in hardware. */ # define TV_FUSE_STATE_DISABLED (2 << 4) -/** Normal operation */ +/* Normal operation */ # define TV_TEST_MODE_NORMAL (0 << 0) -/** Encoder test pattern 1 - combo pattern */ +/* Encoder test pattern 1 - combo pattern */ # define TV_TEST_MODE_PATTERN_1 (1 << 0) -/** Encoder test pattern 2 - full screen vertical 75% color bars */ +/* Encoder test pattern 2 - full screen vertical 75% color bars */ # define TV_TEST_MODE_PATTERN_2 (2 << 0) -/** Encoder test pattern 3 - full screen horizontal 75% color bars */ +/* Encoder test pattern 3 - full screen horizontal 75% color bars */ # define TV_TEST_MODE_PATTERN_3 (3 << 0) -/** Encoder test pattern 4 - random noise */ +/* Encoder test pattern 4 - random noise */ # define TV_TEST_MODE_PATTERN_4 (4 << 0) -/** Encoder test pattern 5 - linear color ramps */ +/* Encoder test pattern 5 - linear color ramps */ # define TV_TEST_MODE_PATTERN_5 (5 << 0) -/** +/* * This test mode forces the DACs to 50% of full output. * * This is used for load detection in combination with TVDAC_SENSE_MASK @@ -2577,35 +2983,35 @@ #define TV_DAC 0x68004 # define TV_DAC_SAVE 0x00ffff00 -/** +/* * Reports that DAC state change logic has reported change (RO). * * This gets cleared when TV_DAC_STATE_EN is cleared */ # define TVDAC_STATE_CHG (1 << 31) # define TVDAC_SENSE_MASK (7 << 28) -/** Reports that DAC A voltage is above the detect threshold */ +/* Reports that DAC A voltage is above the detect threshold */ # define TVDAC_A_SENSE (1 << 30) -/** Reports that DAC B voltage is above the detect threshold */ +/* Reports that DAC B voltage is above the detect threshold */ # define TVDAC_B_SENSE (1 << 29) -/** Reports that DAC C voltage is above the detect threshold */ +/* Reports that DAC C voltage is above the detect threshold */ # define TVDAC_C_SENSE (1 << 28) -/** +/* * Enables DAC state detection logic, for load-based TV detection. * * The PLL of the chosen pipe (in TV_CTL) must be running, and the encoder set * to off, for load detection to work. */ # define TVDAC_STATE_CHG_EN (1 << 27) -/** Sets the DAC A sense value to high */ +/* Sets the DAC A sense value to high */ # define TVDAC_A_SENSE_CTL (1 << 26) -/** Sets the DAC B sense value to high */ +/* Sets the DAC B sense value to high */ # define TVDAC_B_SENSE_CTL (1 << 25) -/** Sets the DAC C sense value to high */ +/* Sets the DAC C sense value to high */ # define TVDAC_C_SENSE_CTL (1 << 24) -/** Overrides the ENC_ENABLE and DAC voltage levels */ +/* Overrides the ENC_ENABLE and DAC voltage levels */ # define DAC_CTL_OVERRIDE (1 << 7) -/** Sets the slew rate. Must be preserved in software */ +/* Sets the slew rate. Must be preserved in software */ # define ENC_TVDAC_SLEW_FAST (1 << 6) # define DAC_A_1_3_V (0 << 4) # define DAC_A_1_1_V (1 << 4) @@ -2620,7 +3026,7 @@ # define DAC_C_0_7_V (2 << 0) # define DAC_C_MASK (3 << 0) -/** +/* * CSC coefficients are stored in a floating point format with 9 bits of * mantissa and 2 or 3 bits of exponent. The exponent is represented as 2**-n, * where 2-bit exponents are unsigned n, and 3-bit exponents are signed n with @@ -2635,7 +3041,7 @@ #define TV_CSC_Y2 0x68014 # define TV_BY_MASK 0x07ff0000 # define TV_BY_SHIFT 16 -/** +/* * Y attenuation for component video. * * Stored in 1.9 fixed point. @@ -2652,7 +3058,7 @@ #define TV_CSC_U2 0x6801c # define TV_BU_MASK 0x07ff0000 # define TV_BU_SHIFT 16 -/** +/* * U attenuation for component video. * * Stored in 1.9 fixed point. @@ -2669,7 +3075,7 @@ #define TV_CSC_V2 0x68024 # define TV_BV_MASK 0x07ff0000 # define TV_BV_SHIFT 16 -/** +/* * V attenuation for component video. * * Stored in 1.9 fixed point. @@ -2678,74 +3084,74 @@ # define TV_AV_SHIFT 0 #define TV_CLR_KNOBS 0x68028 -/** 2s-complement brightness adjustment */ +/* 2s-complement brightness adjustment */ # define TV_BRIGHTNESS_MASK 0xff000000 # define TV_BRIGHTNESS_SHIFT 24 -/** Contrast adjustment, as a 2.6 unsigned floating point number */ +/* Contrast adjustment, as a 2.6 unsigned floating point number */ # define TV_CONTRAST_MASK 0x00ff0000 # define TV_CONTRAST_SHIFT 16 -/** Saturation adjustment, as a 2.6 unsigned floating point number */ +/* Saturation adjustment, as a 2.6 unsigned floating point number */ # define TV_SATURATION_MASK 0x0000ff00 # define TV_SATURATION_SHIFT 8 -/** Hue adjustment, as an integer phase angle in degrees */ +/* Hue adjustment, as an integer phase angle in degrees */ # define TV_HUE_MASK 0x000000ff # define TV_HUE_SHIFT 0 #define TV_CLR_LEVEL 0x6802c -/** Controls the DAC level for black */ +/* Controls the DAC level for black */ # define TV_BLACK_LEVEL_MASK 0x01ff0000 # define TV_BLACK_LEVEL_SHIFT 16 -/** Controls the DAC level for blanking */ +/* Controls the DAC level for blanking */ # define TV_BLANK_LEVEL_MASK 0x000001ff # define TV_BLANK_LEVEL_SHIFT 0 #define TV_H_CTL_1 0x68030 -/** Number of pixels in the hsync. */ +/* Number of pixels in the hsync. */ # define TV_HSYNC_END_MASK 0x1fff0000 # define TV_HSYNC_END_SHIFT 16 -/** Total number of pixels minus one in the line (display and blanking). */ +/* Total number of pixels minus one in the line (display and blanking). */ # define TV_HTOTAL_MASK 0x00001fff # define TV_HTOTAL_SHIFT 0 #define TV_H_CTL_2 0x68034 -/** Enables the colorburst (needed for non-component color) */ +/* Enables the colorburst (needed for non-component color) */ # define TV_BURST_ENA (1 << 31) -/** Offset of the colorburst from the start of hsync, in pixels minus one. */ +/* Offset of the colorburst from the start of hsync, in pixels minus one. */ # define TV_HBURST_START_SHIFT 16 # define TV_HBURST_START_MASK 0x1fff0000 -/** Length of the colorburst */ +/* Length of the colorburst */ # define TV_HBURST_LEN_SHIFT 0 # define TV_HBURST_LEN_MASK 0x0001fff #define TV_H_CTL_3 0x68038 -/** End of hblank, measured in pixels minus one from start of hsync */ +/* End of hblank, measured in pixels minus one from start of hsync */ # define TV_HBLANK_END_SHIFT 16 # define TV_HBLANK_END_MASK 0x1fff0000 -/** Start of hblank, measured in pixels minus one from start of hsync */ +/* Start of hblank, measured in pixels minus one from start of hsync */ # define TV_HBLANK_START_SHIFT 0 # define TV_HBLANK_START_MASK 0x0001fff #define TV_V_CTL_1 0x6803c -/** XXX */ +/* XXX */ # define TV_NBR_END_SHIFT 16 # define TV_NBR_END_MASK 0x07ff0000 -/** XXX */ +/* XXX */ # define TV_VI_END_F1_SHIFT 8 # define TV_VI_END_F1_MASK 0x00003f00 -/** XXX */ +/* XXX */ # define TV_VI_END_F2_SHIFT 0 # define TV_VI_END_F2_MASK 0x0000003f #define TV_V_CTL_2 0x68040 -/** Length of vsync, in half lines */ +/* Length of vsync, in half lines */ # define TV_VSYNC_LEN_MASK 0x07ff0000 # define TV_VSYNC_LEN_SHIFT 16 -/** Offset of the start of vsync in field 1, measured in one less than the +/* Offset of the start of vsync in field 1, measured in one less than the * number of half lines. */ # define TV_VSYNC_START_F1_MASK 0x00007f00 # define TV_VSYNC_START_F1_SHIFT 8 -/** +/* * Offset of the start of vsync in field 2, measured in one less than the * number of half lines. */ @@ -2753,17 +3159,17 @@ # define TV_VSYNC_START_F2_SHIFT 0 #define TV_V_CTL_3 0x68044 -/** Enables generation of the equalization signal */ +/* Enables generation of the equalization signal */ # define TV_EQUAL_ENA (1 << 31) -/** Length of vsync, in half lines */ +/* Length of vsync, in half lines */ # define TV_VEQ_LEN_MASK 0x007f0000 # define TV_VEQ_LEN_SHIFT 16 -/** Offset of the start of equalization in field 1, measured in one less than +/* Offset of the start of equalization in field 1, measured in one less than * the number of half lines. */ # define TV_VEQ_START_F1_MASK 0x0007f00 # define TV_VEQ_START_F1_SHIFT 8 -/** +/* * Offset of the start of equalization in field 2, measured in one less than * the number of half lines. */ @@ -2771,13 +3177,13 @@ # define TV_VEQ_START_F2_SHIFT 0 #define TV_V_CTL_4 0x68048 -/** +/* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F1_MASK 0x003f0000 # define TV_VBURST_START_F1_SHIFT 16 -/** +/* * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ @@ -2785,13 +3191,13 @@ # define TV_VBURST_END_F1_SHIFT 0 #define TV_V_CTL_5 0x6804c -/** +/* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F2_MASK 0x003f0000 # define TV_VBURST_START_F2_SHIFT 16 -/** +/* * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ @@ -2799,13 +3205,13 @@ # define TV_VBURST_END_F2_SHIFT 0 #define TV_V_CTL_6 0x68050 -/** +/* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F3_MASK 0x003f0000 # define TV_VBURST_START_F3_SHIFT 16 -/** +/* * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ @@ -2813,13 +3219,13 @@ # define TV_VBURST_END_F3_SHIFT 0 #define TV_V_CTL_7 0x68054 -/** +/* * Offset to start of vertical colorburst, measured in one less than the * number of lines from vertical start. */ # define TV_VBURST_START_F4_MASK 0x003f0000 # define TV_VBURST_START_F4_SHIFT 16 -/** +/* * Offset to the end of vertical colorburst, measured in one less than the * number of lines from the start of NBR. */ @@ -2827,56 +3233,56 @@ # define TV_VBURST_END_F4_SHIFT 0 #define TV_SC_CTL_1 0x68060 -/** Turns on the first subcarrier phase generation DDA */ +/* Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA1_EN (1 << 31) -/** Turns on the first subcarrier phase generation DDA */ +/* Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA2_EN (1 << 30) -/** Turns on the first subcarrier phase generation DDA */ +/* Turns on the first subcarrier phase generation DDA */ # define TV_SC_DDA3_EN (1 << 29) -/** Sets the subcarrier DDA to reset frequency every other field */ +/* Sets the subcarrier DDA to reset frequency every other field */ # define TV_SC_RESET_EVERY_2 (0 << 24) -/** Sets the subcarrier DDA to reset frequency every fourth field */ +/* Sets the subcarrier DDA to reset frequency every fourth field */ # define TV_SC_RESET_EVERY_4 (1 << 24) -/** Sets the subcarrier DDA to reset frequency every eighth field */ +/* Sets the subcarrier DDA to reset frequency every eighth field */ # define TV_SC_RESET_EVERY_8 (2 << 24) -/** Sets the subcarrier DDA to never reset the frequency */ +/* Sets the subcarrier DDA to never reset the frequency */ # define TV_SC_RESET_NEVER (3 << 24) -/** Sets the peak amplitude of the colorburst.*/ +/* Sets the peak amplitude of the colorburst.*/ # define TV_BURST_LEVEL_MASK 0x00ff0000 # define TV_BURST_LEVEL_SHIFT 16 -/** Sets the increment of the first subcarrier phase generation DDA */ +/* Sets the increment of the first subcarrier phase generation DDA */ # define TV_SCDDA1_INC_MASK 0x00000fff # define TV_SCDDA1_INC_SHIFT 0 #define TV_SC_CTL_2 0x68064 -/** Sets the rollover for the second subcarrier phase generation DDA */ +/* Sets the rollover for the second subcarrier phase generation DDA */ # define TV_SCDDA2_SIZE_MASK 0x7fff0000 # define TV_SCDDA2_SIZE_SHIFT 16 -/** Sets the increent of the second subcarrier phase generation DDA */ +/* Sets the increent of the second subcarrier phase generation DDA */ # define TV_SCDDA2_INC_MASK 0x00007fff # define TV_SCDDA2_INC_SHIFT 0 #define TV_SC_CTL_3 0x68068 -/** Sets the rollover for the third subcarrier phase generation DDA */ +/* Sets the rollover for the third subcarrier phase generation DDA */ # define TV_SCDDA3_SIZE_MASK 0x7fff0000 # define TV_SCDDA3_SIZE_SHIFT 16 -/** Sets the increent of the third subcarrier phase generation DDA */ +/* Sets the increent of the third subcarrier phase generation DDA */ # define TV_SCDDA3_INC_MASK 0x00007fff # define TV_SCDDA3_INC_SHIFT 0 #define TV_WIN_POS 0x68070 -/** X coordinate of the display from the start of horizontal active */ +/* X coordinate of the display from the start of horizontal active */ # define TV_XPOS_MASK 0x1fff0000 # define TV_XPOS_SHIFT 16 -/** Y coordinate of the display from the start of vertical active (NBR) */ +/* Y coordinate of the display from the start of vertical active (NBR) */ # define TV_YPOS_MASK 0x00000fff # define TV_YPOS_SHIFT 0 #define TV_WIN_SIZE 0x68074 -/** Horizontal size of the display window, measured in pixels*/ +/* Horizontal size of the display window, measured in pixels*/ # define TV_XSIZE_MASK 0x1fff0000 # define TV_XSIZE_SHIFT 16 -/** +/* * Vertical size of the display window, measured in pixels. * * Must be even for interlaced modes. @@ -2885,28 +3291,28 @@ # define TV_YSIZE_SHIFT 0 #define TV_FILTER_CTL_1 0x68080 -/** +/* * Enables automatic scaling calculation. * * If set, the rest of the registers are ignored, and the calculated values can * be read back from the register. */ # define TV_AUTO_SCALE (1 << 31) -/** +/* * Disables the vertical filter. * * This is required on modes more than 1024 pixels wide */ # define TV_V_FILTER_BYPASS (1 << 29) -/** Enables adaptive vertical filtering */ +/* Enables adaptive vertical filtering */ # define TV_VADAPT (1 << 28) # define TV_VADAPT_MODE_MASK (3 << 26) -/** Selects the least adaptive vertical filtering mode */ +/* Selects the least adaptive vertical filtering mode */ # define TV_VADAPT_MODE_LEAST (0 << 26) -/** Selects the moderately adaptive vertical filtering mode */ +/* Selects the moderately adaptive vertical filtering mode */ # define TV_VADAPT_MODE_MODERATE (1 << 26) -/** Selects the most adaptive vertical filtering mode */ +/* Selects the most adaptive vertical filtering mode */ # define TV_VADAPT_MODE_MOST (3 << 26) -/** +/* * Sets the horizontal scaling factor. * * This should be the fractional part of the horizontal scaling factor divided @@ -2918,14 +3324,14 @@ # define TV_HSCALE_FRAC_SHIFT 0 #define TV_FILTER_CTL_2 0x68084 -/** +/* * Sets the integer part of the 3.15 fixed-point vertical scaling factor. * * TV_VSCALE should be (src height - 1) / ((interlace * dest height) - 1) */ # define TV_VSCALE_INT_MASK 0x00038000 # define TV_VSCALE_INT_SHIFT 15 -/** +/* * Sets the fractional part of the 3.15 fixed-point vertical scaling factor. * * \sa TV_VSCALE_INT_MASK @@ -2934,7 +3340,7 @@ # define TV_VSCALE_FRAC_SHIFT 0 #define TV_FILTER_CTL_3 0x68088 -/** +/* * Sets the integer part of the 3.15 fixed-point vertical scaling factor. * * TV_VSCALE should be (src height - 1) / (1/4 * (dest height - 1)) @@ -2943,7 +3349,7 @@ */ # define TV_VSCALE_IP_INT_MASK 0x00038000 # define TV_VSCALE_IP_INT_SHIFT 15 -/** +/* * Sets the fractional part of the 3.15 fixed-point vertical scaling factor. * * For progressive modes, TV_VSCALE_IP_INT should be set to zeroes. @@ -2955,26 +3361,26 @@ #define TV_CC_CONTROL 0x68090 # define TV_CC_ENABLE (1 << 31) -/** +/* * Specifies which field to send the CC data in. * * CC data is usually sent in field 0. */ # define TV_CC_FID_MASK (1 << 27) # define TV_CC_FID_SHIFT 27 -/** Sets the horizontal position of the CC data. Usually 135. */ +/* Sets the horizontal position of the CC data. Usually 135. */ # define TV_CC_HOFF_MASK 0x03ff0000 # define TV_CC_HOFF_SHIFT 16 -/** Sets the vertical position of the CC data. Usually 21 */ +/* Sets the vertical position of the CC data. Usually 21 */ # define TV_CC_LINE_MASK 0x0000003f # define TV_CC_LINE_SHIFT 0 #define TV_CC_DATA 0x68094 # define TV_CC_RDY (1 << 31) -/** Second word of CC data to be transmitted. */ +/* Second word of CC data to be transmitted. */ # define TV_CC_DATA_2_MASK 0x007f0000 # define TV_CC_DATA_2_SHIFT 16 -/** First word of CC data to be transmitted. */ +/* First word of CC data to be transmitted. */ # define TV_CC_DATA_1_MASK 0x0000007f # define TV_CC_DATA_1_SHIFT 0 @@ -2996,6 +3402,8 @@ #define DP_PORT_EN (1 << 31) #define DP_PIPEB_SELECT (1 << 30) #define DP_PIPE_MASK (1 << 30) +#define DP_PIPE_SELECT_CHV(pipe) ((pipe) << 16) +#define DP_PIPE_MASK_CHV (3 << 16) /* Link training mode - select a suitable mode for each stage */ #define DP_LINK_TRAIN_PAT_1 (0 << 28) @@ -3043,32 +3451,32 @@ #define DP_PLL_FREQ_160MHZ (1 << 16) #define DP_PLL_FREQ_MASK (3 << 16) -/** locked once port is enabled */ +/* locked once port is enabled */ #define DP_PORT_REVERSAL (1 << 15) /* eDP */ #define DP_PLL_ENABLE (1 << 14) -/** sends the clock on lane 15 of the PEG for debug */ +/* sends the clock on lane 15 of the PEG for debug */ #define DP_CLOCK_OUTPUT_ENABLE (1 << 13) #define DP_SCRAMBLING_DISABLE (1 << 12) #define DP_SCRAMBLING_DISABLE_IRONLAKE (1 << 7) -/** limit RGB values to avoid confusing TVs */ +/* limit RGB values to avoid confusing TVs */ #define DP_COLOR_RANGE_16_235 (1 << 8) -/** Turn on the audio link */ +/* Turn on the audio link */ #define DP_AUDIO_OUTPUT_ENABLE (1 << 6) -/** vs and hs sync polarity */ +/* vs and hs sync polarity */ #define DP_SYNC_VS_HIGH (1 << 4) #define DP_SYNC_HS_HIGH (1 << 3) -/** A fantasy */ +/* A fantasy */ #define DP_DETECTED (1 << 2) -/** The aux channel provides a way to talk to the +/* The aux channel provides a way to talk to the * signal sink for DDC etc. Max packet size supported * is 20 bytes in each direction, hence the 5 fixed * data registers @@ -3178,10 +3586,10 @@ /* Display & cursor control */ /* Pipe A */ -#define _PIPEADSL (dev_priv->info->display_mmio_offset + 0x70000) +#define _PIPEADSL 0x70000 #define DSL_LINEMASK_GEN2 0x00000fff #define DSL_LINEMASK_GEN3 0x00001fff -#define _PIPEACONF (dev_priv->info->display_mmio_offset + 0x70008) +#define _PIPEACONF 0x70008 #define PIPECONF_ENABLE (1<<31) #define PIPECONF_DISABLE 0 #define PIPECONF_DOUBLE_WIDE (1<<30) @@ -3211,6 +3619,7 @@ #define PIPECONF_INTERLACED_DBL_ILK (4 << 21) /* ilk/snb only */ #define PIPECONF_PFIT_PF_INTERLACED_DBL_ILK (5 << 21) /* ilk/snb only */ #define PIPECONF_INTERLACE_MODE_MASK (7 << 21) +#define PIPECONF_EDP_RR_MODE_SWITCH (1 << 20) #define PIPECONF_CXSR_DOWNCLOCK (1<<16) #define PIPECONF_COLOR_RANGE_SELECT (1 << 13) #define PIPECONF_BPC_MASK (0x7 << 5) @@ -3224,11 +3633,12 @@ #define PIPECONF_DITHER_TYPE_ST1 (1<<2) #define PIPECONF_DITHER_TYPE_ST2 (2<<2) #define PIPECONF_DITHER_TYPE_TEMP (3<<2) -#define _PIPEASTAT (dev_priv->info->display_mmio_offset + 0x70024) +#define _PIPEASTAT 0x70024 #define PIPE_FIFO_UNDERRUN_STATUS (1UL<<31) -#define SPRITE1_FLIPDONE_INT_EN_VLV (1UL<<30) +#define SPRITE1_FLIP_DONE_INT_EN_VLV (1UL<<30) #define PIPE_CRC_ERROR_ENABLE (1UL<<29) #define PIPE_CRC_DONE_ENABLE (1UL<<28) +#define PERF_COUNTER2_INTERRUPT_EN (1UL<<27) #define PIPE_GMBUS_EVENT_ENABLE (1UL<<27) #define PLANE_FLIP_DONE_INT_EN_VLV (1UL<<26) #define PIPE_HOTPLUG_INTERRUPT_ENABLE (1UL<<26) @@ -3239,35 +3649,63 @@ #define PIPE_LEGACY_BLC_EVENT_ENABLE (1UL<<22) #define PIPE_ODD_FIELD_INTERRUPT_ENABLE (1UL<<21) #define PIPE_EVEN_FIELD_INTERRUPT_ENABLE (1UL<<20) +#define PIPE_B_PSR_INTERRUPT_ENABLE_VLV (1UL<<19) +#define PERF_COUNTER_INTERRUPT_EN (1UL<<19) #define PIPE_HOTPLUG_TV_INTERRUPT_ENABLE (1UL<<18) /* pre-965 */ #define PIPE_START_VBLANK_INTERRUPT_ENABLE (1UL<<18) /* 965 or later */ +#define PIPE_FRAMESTART_INTERRUPT_ENABLE (1UL<<17) #define PIPE_VBLANK_INTERRUPT_ENABLE (1UL<<17) #define PIPEA_HBLANK_INT_EN_VLV (1UL<<16) #define PIPE_OVERLAY_UPDATED_ENABLE (1UL<<16) -#define SPRITE1_FLIPDONE_INT_STATUS_VLV (1UL<<15) -#define SPRITE0_FLIPDONE_INT_STATUS_VLV (1UL<<14) +#define SPRITE1_FLIP_DONE_INT_STATUS_VLV (1UL<<15) +#define SPRITE0_FLIP_DONE_INT_STATUS_VLV (1UL<<14) #define PIPE_CRC_ERROR_INTERRUPT_STATUS (1UL<<13) #define PIPE_CRC_DONE_INTERRUPT_STATUS (1UL<<12) +#define PERF_COUNTER2_INTERRUPT_STATUS (1UL<<11) #define PIPE_GMBUS_INTERRUPT_STATUS (1UL<<11) -#define PLANE_FLIPDONE_INT_STATUS_VLV (1UL<<10) +#define PLANE_FLIP_DONE_INT_STATUS_VLV (1UL<<10) #define PIPE_HOTPLUG_INTERRUPT_STATUS (1UL<<10) #define PIPE_VSYNC_INTERRUPT_STATUS (1UL<<9) #define PIPE_DISPLAY_LINE_COMPARE_STATUS (1UL<<8) #define PIPE_DPST_EVENT_STATUS (1UL<<7) #define PIPE_LEGACY_BLC_EVENT_STATUS (1UL<<6) +#define PIPE_A_PSR_STATUS_VLV (1UL<<6) +#define PIPE_LEGACY_BLC_EVENT_STATUS (1UL<<6) #define PIPE_ODD_FIELD_INTERRUPT_STATUS (1UL<<5) #define PIPE_EVEN_FIELD_INTERRUPT_STATUS (1UL<<4) +#define PIPE_B_PSR_STATUS_VLV (1UL<<3) +#define PERF_COUNTER_INTERRUPT_STATUS (1UL<<3) #define PIPE_HOTPLUG_TV_INTERRUPT_STATUS (1UL<<2) /* pre-965 */ #define PIPE_START_VBLANK_INTERRUPT_STATUS (1UL<<2) /* 965 or later */ +#define PIPE_FRAMESTART_INTERRUPT_STATUS (1UL<<1) #define PIPE_VBLANK_INTERRUPT_STATUS (1UL<<1) +#define PIPE_HBLANK_INT_STATUS (1UL<<0) #define PIPE_OVERLAY_UPDATED_STATUS (1UL<<0) -#define PIPESRC(pipe) _PIPE(pipe, _PIPEASRC, _PIPEBSRC) -#define PIPECONF(tran) _TRANSCODER(tran, _PIPEACONF, _PIPEBCONF) -#define PIPEDSL(pipe) _PIPE(pipe, _PIPEADSL, _PIPEBDSL) -#define PIPEFRAME(pipe) _PIPE(pipe, _PIPEAFRAMEHIGH, _PIPEBFRAMEHIGH) -#define PIPEFRAMEPIXEL(pipe) _PIPE(pipe, _PIPEAFRAMEPIXEL, _PIPEBFRAMEPIXEL) -#define PIPESTAT(pipe) _PIPE(pipe, _PIPEASTAT, _PIPEBSTAT) +#define PIPESTAT_INT_ENABLE_MASK 0x7fff0000 +#define PIPESTAT_INT_STATUS_MASK 0x0000ffff + +#define PIPE_A_OFFSET 0x70000 +#define PIPE_B_OFFSET 0x71000 +#define PIPE_C_OFFSET 0x72000 +#define CHV_PIPE_C_OFFSET 0x74000 +/* + * There's actually no pipe EDP. Some pipe registers have + * simply shifted from the pipe to the transcoder, while + * keeping their original offset. Thus we need PIPE_EDP_OFFSET + * to access such registers in transcoder EDP. + */ +#define PIPE_EDP_OFFSET 0x7f000 + +#define _PIPE2(pipe, reg) (dev_priv->info.pipe_offsets[pipe] - \ + dev_priv->info.pipe_offsets[PIPE_A] + (reg) + \ + dev_priv->info.display_mmio_offset) + +#define PIPECONF(pipe) _PIPE2(pipe, _PIPEACONF) +#define PIPEDSL(pipe) _PIPE2(pipe, _PIPEADSL) +#define PIPEFRAME(pipe) _PIPE2(pipe, _PIPEAFRAMEHIGH) +#define PIPEFRAMEPIXEL(pipe) _PIPE2(pipe, _PIPEAFRAMEPIXEL) +#define PIPESTAT(pipe) _PIPE2(pipe, _PIPEASTAT) #define _PIPE_MISC_A 0x70030 #define _PIPE_MISC_B 0x71030 @@ -3279,23 +3717,34 @@ #define PIPEMISC_DITHER_ENABLE (1<<4) #define PIPEMISC_DITHER_TYPE_MASK (3<<2) #define PIPEMISC_DITHER_TYPE_SP (0<<2) -#define PIPEMISC(pipe) _PIPE(pipe, _PIPE_MISC_A, _PIPE_MISC_B) +#define PIPEMISC(pipe) _PIPE2(pipe, _PIPE_MISC_A) #define VLV_DPFLIPSTAT (VLV_DISPLAY_BASE + 0x70028) #define PIPEB_LINE_COMPARE_INT_EN (1<<29) #define PIPEB_HLINE_INT_EN (1<<28) #define PIPEB_VBLANK_INT_EN (1<<27) -#define SPRITED_FLIPDONE_INT_EN (1<<26) -#define SPRITEC_FLIPDONE_INT_EN (1<<25) -#define PLANEB_FLIPDONE_INT_EN (1<<24) +#define SPRITED_FLIP_DONE_INT_EN (1<<26) +#define SPRITEC_FLIP_DONE_INT_EN (1<<25) +#define PLANEB_FLIP_DONE_INT_EN (1<<24) +#define PIPE_PSR_INT_EN (1<<22) #define PIPEA_LINE_COMPARE_INT_EN (1<<21) #define PIPEA_HLINE_INT_EN (1<<20) #define PIPEA_VBLANK_INT_EN (1<<19) -#define SPRITEB_FLIPDONE_INT_EN (1<<18) -#define SPRITEA_FLIPDONE_INT_EN (1<<17) +#define SPRITEB_FLIP_DONE_INT_EN (1<<18) +#define SPRITEA_FLIP_DONE_INT_EN (1<<17) #define PLANEA_FLIPDONE_INT_EN (1<<16) - -#define DPINVGTT (VLV_DISPLAY_BASE + 0x7002c) /* VLV only */ +#define PIPEC_LINE_COMPARE_INT_EN (1<<13) +#define PIPEC_HLINE_INT_EN (1<<12) +#define PIPEC_VBLANK_INT_EN (1<<11) +#define SPRITEF_FLIPDONE_INT_EN (1<<10) +#define SPRITEE_FLIPDONE_INT_EN (1<<9) +#define PLANEC_FLIPDONE_INT_EN (1<<8) + +#define DPINVGTT (VLV_DISPLAY_BASE + 0x7002c) /* VLV/CHV only */ +#define SPRITEF_INVALID_GTT_INT_EN (1<<27) +#define SPRITEE_INVALID_GTT_INT_EN (1<<26) +#define PLANEC_INVALID_GTT_INT_EN (1<<25) +#define CURSORC_INVALID_GTT_INT_EN (1<<24) #define CURSORB_INVALID_GTT_INT_EN (1<<23) #define CURSORA_INVALID_GTT_INT_EN (1<<22) #define SPRITED_INVALID_GTT_INT_EN (1<<21) @@ -3305,6 +3754,11 @@ #define SPRITEA_INVALID_GTT_INT_EN (1<<17) #define PLANEA_INVALID_GTT_INT_EN (1<<16) #define DPINVGTT_EN_MASK 0xff0000 +#define DPINVGTT_EN_MASK_CHV 0xfff0000 +#define SPRITEF_INVALID_GTT_STATUS (1<<11) +#define SPRITEE_INVALID_GTT_STATUS (1<<10) +#define PLANEC_INVALID_GTT_STATUS (1<<9) +#define CURSORC_INVALID_GTT_STATUS (1<<8) #define CURSORB_INVALID_GTT_STATUS (1<<7) #define CURSORA_INVALID_GTT_STATUS (1<<6) #define SPRITED_INVALID_GTT_STATUS (1<<5) @@ -3314,6 +3768,7 @@ #define SPRITEA_INVALID_GTT_STATUS (1<<1) #define PLANEA_INVALID_GTT_STATUS (1<<0) #define DPINVGTT_STATUS_MASK 0xff +#define DPINVGTT_STATUS_MASK_CHV 0xfff #define DSPARB 0x70030 #define DSPARB_CSTART_MASK (0x7f << 7) @@ -3323,7 +3778,7 @@ #define DSPARB_BEND_SHIFT 9 /* on 855 */ #define DSPARB_AEND_SHIFT 0 -#define DSPFW1 (dev_priv->info->display_mmio_offset + 0x70034) +#define DSPFW1 (dev_priv->info.display_mmio_offset + 0x70034) #define DSPFW_SR_SHIFT 23 #define DSPFW_SR_MASK (0x1ff<<23) #define DSPFW_CURSORB_SHIFT 16 @@ -3331,11 +3786,11 @@ #define DSPFW_PLANEB_SHIFT 8 #define DSPFW_PLANEB_MASK (0x7f<<8) #define DSPFW_PLANEA_MASK (0x7f) -#define DSPFW2 (dev_priv->info->display_mmio_offset + 0x70038) +#define DSPFW2 (dev_priv->info.display_mmio_offset + 0x70038) #define DSPFW_CURSORA_MASK 0x00003f00 #define DSPFW_CURSORA_SHIFT 8 #define DSPFW_PLANEC_MASK (0x7f) -#define DSPFW3 (dev_priv->info->display_mmio_offset + 0x7003c) +#define DSPFW3 (dev_priv->info.display_mmio_offset + 0x7003c) #define DSPFW_HPLL_SR_EN (1<<31) #define DSPFW_CURSOR_SR_SHIFT 24 #define PINEVIEW_SELF_REFRESH_EN (1<<30) @@ -3343,8 +3798,8 @@ #define DSPFW_HPLL_CURSOR_SHIFT 16 #define DSPFW_HPLL_CURSOR_MASK (0x3f<<16) #define DSPFW_HPLL_SR_MASK (0x1ff) -#define DSPFW4 (dev_priv->info->display_mmio_offset + 0x70070) -#define DSPFW7 (dev_priv->info->display_mmio_offset + 0x7007c) +#define DSPFW4 (dev_priv->info.display_mmio_offset + 0x70070) +#define DSPFW7 (dev_priv->info.display_mmio_offset + 0x7007c) /* drain latency register values*/ #define DRAIN_LATENCY_PRECISION_32 32 @@ -3353,14 +3808,43 @@ #define DDL_CURSORA_PRECISION_32 (1<<31) #define DDL_CURSORA_PRECISION_16 (0<<31) #define DDL_CURSORA_SHIFT 24 +#define DDL_SPRITEB_PRECISION_32 (1<<23) +#define DDL_SPRITEB_PRECISION_16 (0<<23) +#define DDL_SPRITEB_SHIFT 16 +#define DDL_SPRITEA_PRECISION_32 (1<<15) +#define DDL_SPRITEA_PRECISION_16 (0<<15) +#define DDL_SPRITEA_SHIFT 8 #define DDL_PLANEA_PRECISION_32 (1<<7) #define DDL_PLANEA_PRECISION_16 (0<<7) +#define DDL_PLANEA_SHIFT 0 + #define VLV_DDL2 (VLV_DISPLAY_BASE + 0x70054) #define DDL_CURSORB_PRECISION_32 (1<<31) #define DDL_CURSORB_PRECISION_16 (0<<31) #define DDL_CURSORB_SHIFT 24 +#define DDL_SPRITED_PRECISION_32 (1<<23) +#define DDL_SPRITED_PRECISION_16 (0<<23) +#define DDL_SPRITED_SHIFT 16 +#define DDL_SPRITEC_PRECISION_32 (1<<15) +#define DDL_SPRITEC_PRECISION_16 (0<<15) +#define DDL_SPRITEC_SHIFT 8 #define DDL_PLANEB_PRECISION_32 (1<<7) #define DDL_PLANEB_PRECISION_16 (0<<7) +#define DDL_PLANEB_SHIFT 0 + +#define VLV_DDL3 (VLV_DISPLAY_BASE + 0x70058) +#define DDL_CURSORC_PRECISION_32 (1<<31) +#define DDL_CURSORC_PRECISION_16 (0<<31) +#define DDL_CURSORC_SHIFT 24 +#define DDL_SPRITEF_PRECISION_32 (1<<23) +#define DDL_SPRITEF_PRECISION_16 (0<<23) +#define DDL_SPRITEF_SHIFT 16 +#define DDL_SPRITEE_PRECISION_32 (1<<15) +#define DDL_SPRITEE_PRECISION_16 (0<<15) +#define DDL_SPRITEE_SHIFT 8 +#define DDL_PLANEC_PRECISION_32 (1<<7) +#define DDL_PLANEC_PRECISION_16 (0<<7) +#define DDL_PLANEC_SHIFT 0 /* FIFO watermark sizes etc */ #define G4X_FIFO_LINE_SIZE 64 @@ -3468,12 +3952,13 @@ #define PIPE_PIXEL_MASK 0x00ffffff #define PIPE_PIXEL_SHIFT 0 /* GM45+ just has to be different */ -#define _PIPEA_FRMCOUNT_GM45 (dev_priv->info->display_mmio_offset + 0x70040) -#define _PIPEA_FLIPCOUNT_GM45 (dev_priv->info->display_mmio_offset + 0x70044) -#define PIPE_FRMCOUNT_GM45(pipe) _PIPE(pipe, _PIPEA_FRMCOUNT_GM45, _PIPEB_FRMCOUNT_GM45) +#define _PIPEA_FRMCOUNT_GM45 0x70040 +#define _PIPEA_FLIPCOUNT_GM45 0x70044 +#define PIPE_FRMCOUNT_GM45(pipe) _PIPE2(pipe, _PIPEA_FRMCOUNT_GM45) +#define PIPE_FLIPCOUNT_GM45(pipe) _PIPE2(pipe, _PIPEA_FLIPCOUNT_GM45) /* Cursor A & B regs */ -#define _CURACNTR (dev_priv->info->display_mmio_offset + 0x70080) +#define _CURACNTR 0x70080 /* Old style CUR*CNTR flags (desktop 8xx) */ #define CURSOR_ENABLE 0x80000000 #define CURSOR_GAMMA_ENABLE 0x40000000 @@ -3489,38 +3974,48 @@ /* New style CUR*CNTR flags */ #define CURSOR_MODE 0x27 #define CURSOR_MODE_DISABLE 0x00 +#define CURSOR_MODE_128_32B_AX 0x02 +#define CURSOR_MODE_256_32B_AX 0x03 #define CURSOR_MODE_64_32B_AX 0x07 +#define CURSOR_MODE_128_ARGB_AX ((1 << 5) | CURSOR_MODE_128_32B_AX) +#define CURSOR_MODE_256_ARGB_AX ((1 << 5) | CURSOR_MODE_256_32B_AX) #define CURSOR_MODE_64_ARGB_AX ((1 << 5) | CURSOR_MODE_64_32B_AX) #define MCURSOR_PIPE_SELECT (1 << 28) #define MCURSOR_PIPE_A 0x00 #define MCURSOR_PIPE_B (1 << 28) #define MCURSOR_GAMMA_ENABLE (1 << 26) #define CURSOR_TRICKLE_FEED_DISABLE (1 << 14) -#define _CURABASE (dev_priv->info->display_mmio_offset + 0x70084) -#define _CURAPOS (dev_priv->info->display_mmio_offset + 0x70088) +#define _CURABASE 0x70084 +#define _CURAPOS 0x70088 #define CURSOR_POS_MASK 0x007FF #define CURSOR_POS_SIGN 0x8000 #define CURSOR_X_SHIFT 0 #define CURSOR_Y_SHIFT 16 #define CURSIZE 0x700a0 -#define _CURBCNTR (dev_priv->info->display_mmio_offset + 0x700c0) -#define _CURBBASE (dev_priv->info->display_mmio_offset + 0x700c4) -#define _CURBPOS (dev_priv->info->display_mmio_offset + 0x700c8) +#define _CURBCNTR 0x700c0 +#define _CURBBASE 0x700c4 +#define _CURBPOS 0x700c8 #define _CURBCNTR_IVB 0x71080 #define _CURBBASE_IVB 0x71084 #define _CURBPOS_IVB 0x71088 -#define CURCNTR(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR) -#define CURBASE(pipe) _PIPE(pipe, _CURABASE, _CURBBASE) -#define CURPOS(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS) +#define _CURSOR2(pipe, reg) (dev_priv->info.cursor_offsets[(pipe)] - \ + dev_priv->info.cursor_offsets[PIPE_A] + (reg) + \ + dev_priv->info.display_mmio_offset) -#define CURCNTR_IVB(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR_IVB) -#define CURBASE_IVB(pipe) _PIPE(pipe, _CURABASE, _CURBBASE_IVB) -#define CURPOS_IVB(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS_IVB) +#define CURCNTR(pipe) _CURSOR2(pipe, _CURACNTR) +#define CURBASE(pipe) _CURSOR2(pipe, _CURABASE) +#define CURPOS(pipe) _CURSOR2(pipe, _CURAPOS) + +#define CURSOR_A_OFFSET 0x70080 +#define CURSOR_B_OFFSET 0x700c0 +#define CHV_CURSOR_C_OFFSET 0x700e0 +#define IVB_CURSOR_B_OFFSET 0x71080 +#define IVB_CURSOR_C_OFFSET 0x72080 /* Display A control */ -#define _DSPACNTR (dev_priv->info->display_mmio_offset + 0x70180) +#define _DSPACNTR 0x70180 #define DISPLAY_PLANE_ENABLE (1<<31) #define DISPLAY_PLANE_DISABLE 0 #define DISPPLANE_GAMMA_ENABLE (1<<30) @@ -3554,25 +4049,25 @@ #define DISPPLANE_STEREO_POLARITY_SECOND (1<<18) #define DISPPLANE_TRICKLE_FEED_DISABLE (1<<14) /* Ironlake */ #define DISPPLANE_TILED (1<<10) -#define _DSPAADDR (dev_priv->info->display_mmio_offset + 0x70184) -#define _DSPASTRIDE (dev_priv->info->display_mmio_offset + 0x70188) -#define _DSPAPOS (dev_priv->info->display_mmio_offset + 0x7018C) /* reserved */ -#define _DSPASIZE (dev_priv->info->display_mmio_offset + 0x70190) -#define _DSPASURF (dev_priv->info->display_mmio_offset + 0x7019C) /* 965+ only */ -#define _DSPATILEOFF (dev_priv->info->display_mmio_offset + 0x701A4) /* 965+ only */ -#define _DSPAOFFSET (dev_priv->info->display_mmio_offset + 0x701A4) /* HSW */ -#define _DSPASURFLIVE (dev_priv->info->display_mmio_offset + 0x701AC) - -#define DSPCNTR(plane) _PIPE(plane, _DSPACNTR, _DSPBCNTR) -#define DSPADDR(plane) _PIPE(plane, _DSPAADDR, _DSPBADDR) -#define DSPSTRIDE(plane) _PIPE(plane, _DSPASTRIDE, _DSPBSTRIDE) -#define DSPPOS(plane) _PIPE(plane, _DSPAPOS, _DSPBPOS) -#define DSPSIZE(plane) _PIPE(plane, _DSPASIZE, _DSPBSIZE) -#define DSPSURF(plane) _PIPE(plane, _DSPASURF, _DSPBSURF) -#define DSPTILEOFF(plane) _PIPE(plane, _DSPATILEOFF, _DSPBTILEOFF) +#define _DSPAADDR 0x70184 +#define _DSPASTRIDE 0x70188 +#define _DSPAPOS 0x7018C /* reserved */ +#define _DSPASIZE 0x70190 +#define _DSPASURF 0x7019C /* 965+ only */ +#define _DSPATILEOFF 0x701A4 /* 965+ only */ +#define _DSPAOFFSET 0x701A4 /* HSW */ +#define _DSPASURFLIVE 0x701AC + +#define DSPCNTR(plane) _PIPE2(plane, _DSPACNTR) +#define DSPADDR(plane) _PIPE2(plane, _DSPAADDR) +#define DSPSTRIDE(plane) _PIPE2(plane, _DSPASTRIDE) +#define DSPPOS(plane) _PIPE2(plane, _DSPAPOS) +#define DSPSIZE(plane) _PIPE2(plane, _DSPASIZE) +#define DSPSURF(plane) _PIPE2(plane, _DSPASURF) +#define DSPTILEOFF(plane) _PIPE2(plane, _DSPATILEOFF) #define DSPLINOFF(plane) DSPADDR(plane) -#define DSPOFFSET(plane) _PIPE(plane, _DSPAOFFSET, _DSPBOFFSET) -#define DSPSURFLIVE(plane) _PIPE(plane, _DSPASURFLIVE, _DSPBSURFLIVE) +#define DSPOFFSET(plane) _PIPE2(plane, _DSPAOFFSET) +#define DSPSURFLIVE(plane) _PIPE2(plane, _DSPASURFLIVE) /* Display/Sprite base address macros */ #define DISP_BASEADDR_MASK (0xfffff000) @@ -3580,44 +4075,44 @@ #define I915_HI_DISPBASE(val) (val & DISP_BASEADDR_MASK) /* VBIOS flags */ -#define SWF00 (dev_priv->info->display_mmio_offset + 0x71410) -#define SWF01 (dev_priv->info->display_mmio_offset + 0x71414) -#define SWF02 (dev_priv->info->display_mmio_offset + 0x71418) -#define SWF03 (dev_priv->info->display_mmio_offset + 0x7141c) -#define SWF04 (dev_priv->info->display_mmio_offset + 0x71420) -#define SWF05 (dev_priv->info->display_mmio_offset + 0x71424) -#define SWF06 (dev_priv->info->display_mmio_offset + 0x71428) -#define SWF10 (dev_priv->info->display_mmio_offset + 0x70410) -#define SWF11 (dev_priv->info->display_mmio_offset + 0x70414) -#define SWF14 (dev_priv->info->display_mmio_offset + 0x71420) -#define SWF30 (dev_priv->info->display_mmio_offset + 0x72414) -#define SWF31 (dev_priv->info->display_mmio_offset + 0x72418) -#define SWF32 (dev_priv->info->display_mmio_offset + 0x7241c) +#define SWF00 (dev_priv->info.display_mmio_offset + 0x71410) +#define SWF01 (dev_priv->info.display_mmio_offset + 0x71414) +#define SWF02 (dev_priv->info.display_mmio_offset + 0x71418) +#define SWF03 (dev_priv->info.display_mmio_offset + 0x7141c) +#define SWF04 (dev_priv->info.display_mmio_offset + 0x71420) +#define SWF05 (dev_priv->info.display_mmio_offset + 0x71424) +#define SWF06 (dev_priv->info.display_mmio_offset + 0x71428) +#define SWF10 (dev_priv->info.display_mmio_offset + 0x70410) +#define SWF11 (dev_priv->info.display_mmio_offset + 0x70414) +#define SWF14 (dev_priv->info.display_mmio_offset + 0x71420) +#define SWF30 (dev_priv->info.display_mmio_offset + 0x72414) +#define SWF31 (dev_priv->info.display_mmio_offset + 0x72418) +#define SWF32 (dev_priv->info.display_mmio_offset + 0x7241c) /* Pipe B */ -#define _PIPEBDSL (dev_priv->info->display_mmio_offset + 0x71000) -#define _PIPEBCONF (dev_priv->info->display_mmio_offset + 0x71008) -#define _PIPEBSTAT (dev_priv->info->display_mmio_offset + 0x71024) +#define _PIPEBDSL (dev_priv->info.display_mmio_offset + 0x71000) +#define _PIPEBCONF (dev_priv->info.display_mmio_offset + 0x71008) +#define _PIPEBSTAT (dev_priv->info.display_mmio_offset + 0x71024) #define _PIPEBFRAMEHIGH 0x71040 #define _PIPEBFRAMEPIXEL 0x71044 -#define _PIPEB_FRMCOUNT_GM45 (dev_priv->info->display_mmio_offset + 0x71040) -#define _PIPEB_FLIPCOUNT_GM45 (dev_priv->info->display_mmio_offset + 0x71044) +#define _PIPEB_FRMCOUNT_GM45 (dev_priv->info.display_mmio_offset + 0x71040) +#define _PIPEB_FLIPCOUNT_GM45 (dev_priv->info.display_mmio_offset + 0x71044) /* Display B control */ -#define _DSPBCNTR (dev_priv->info->display_mmio_offset + 0x71180) +#define _DSPBCNTR (dev_priv->info.display_mmio_offset + 0x71180) #define DISPPLANE_ALPHA_TRANS_ENABLE (1<<15) #define DISPPLANE_ALPHA_TRANS_DISABLE 0 #define DISPPLANE_SPRITE_ABOVE_DISPLAY 0 #define DISPPLANE_SPRITE_ABOVE_OVERLAY (1) -#define _DSPBADDR (dev_priv->info->display_mmio_offset + 0x71184) -#define _DSPBSTRIDE (dev_priv->info->display_mmio_offset + 0x71188) -#define _DSPBPOS (dev_priv->info->display_mmio_offset + 0x7118C) -#define _DSPBSIZE (dev_priv->info->display_mmio_offset + 0x71190) -#define _DSPBSURF (dev_priv->info->display_mmio_offset + 0x7119C) -#define _DSPBTILEOFF (dev_priv->info->display_mmio_offset + 0x711A4) -#define _DSPBOFFSET (dev_priv->info->display_mmio_offset + 0x711A4) -#define _DSPBSURFLIVE (dev_priv->info->display_mmio_offset + 0x711AC) +#define _DSPBADDR (dev_priv->info.display_mmio_offset + 0x71184) +#define _DSPBSTRIDE (dev_priv->info.display_mmio_offset + 0x71188) +#define _DSPBPOS (dev_priv->info.display_mmio_offset + 0x7118C) +#define _DSPBSIZE (dev_priv->info.display_mmio_offset + 0x71190) +#define _DSPBSURF (dev_priv->info.display_mmio_offset + 0x7119C) +#define _DSPBTILEOFF (dev_priv->info.display_mmio_offset + 0x711A4) +#define _DSPBOFFSET (dev_priv->info.display_mmio_offset + 0x711A4) +#define _DSPBSURFLIVE (dev_priv->info.display_mmio_offset + 0x711AC) /* Sprite A control */ #define _DVSACNTR 0x72180 @@ -3866,48 +4361,45 @@ #define FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK 0xff -#define _PIPEA_DATA_M1 (dev_priv->info->display_mmio_offset + 0x60030) +#define _PIPEA_DATA_M1 0x60030 #define PIPE_DATA_M1_OFFSET 0 -#define _PIPEA_DATA_N1 (dev_priv->info->display_mmio_offset + 0x60034) +#define _PIPEA_DATA_N1 0x60034 #define PIPE_DATA_N1_OFFSET 0 -#define _PIPEA_DATA_M2 (dev_priv->info->display_mmio_offset + 0x60038) +#define _PIPEA_DATA_M2 0x60038 #define PIPE_DATA_M2_OFFSET 0 -#define _PIPEA_DATA_N2 (dev_priv->info->display_mmio_offset + 0x6003c) +#define _PIPEA_DATA_N2 0x6003c #define PIPE_DATA_N2_OFFSET 0 -#define _PIPEA_LINK_M1 (dev_priv->info->display_mmio_offset + 0x60040) +#define _PIPEA_LINK_M1 0x60040 #define PIPE_LINK_M1_OFFSET 0 -#define _PIPEA_LINK_N1 (dev_priv->info->display_mmio_offset + 0x60044) +#define _PIPEA_LINK_N1 0x60044 #define PIPE_LINK_N1_OFFSET 0 -#define _PIPEA_LINK_M2 (dev_priv->info->display_mmio_offset + 0x60048) +#define _PIPEA_LINK_M2 0x60048 #define PIPE_LINK_M2_OFFSET 0 -#define _PIPEA_LINK_N2 (dev_priv->info->display_mmio_offset + 0x6004c) +#define _PIPEA_LINK_N2 0x6004c #define PIPE_LINK_N2_OFFSET 0 /* PIPEB timing regs are same start from 0x61000 */ -#define _PIPEB_DATA_M1 (dev_priv->info->display_mmio_offset + 0x61030) -#define _PIPEB_DATA_N1 (dev_priv->info->display_mmio_offset + 0x61034) - -#define _PIPEB_DATA_M2 (dev_priv->info->display_mmio_offset + 0x61038) -#define _PIPEB_DATA_N2 (dev_priv->info->display_mmio_offset + 0x6103c) - -#define _PIPEB_LINK_M1 (dev_priv->info->display_mmio_offset + 0x61040) -#define _PIPEB_LINK_N1 (dev_priv->info->display_mmio_offset + 0x61044) - -#define _PIPEB_LINK_M2 (dev_priv->info->display_mmio_offset + 0x61048) -#define _PIPEB_LINK_N2 (dev_priv->info->display_mmio_offset + 0x6104c) - -#define PIPE_DATA_M1(tran) _TRANSCODER(tran, _PIPEA_DATA_M1, _PIPEB_DATA_M1) -#define PIPE_DATA_N1(tran) _TRANSCODER(tran, _PIPEA_DATA_N1, _PIPEB_DATA_N1) -#define PIPE_DATA_M2(tran) _TRANSCODER(tran, _PIPEA_DATA_M2, _PIPEB_DATA_M2) -#define PIPE_DATA_N2(tran) _TRANSCODER(tran, _PIPEA_DATA_N2, _PIPEB_DATA_N2) -#define PIPE_LINK_M1(tran) _TRANSCODER(tran, _PIPEA_LINK_M1, _PIPEB_LINK_M1) -#define PIPE_LINK_N1(tran) _TRANSCODER(tran, _PIPEA_LINK_N1, _PIPEB_LINK_N1) -#define PIPE_LINK_M2(tran) _TRANSCODER(tran, _PIPEA_LINK_M2, _PIPEB_LINK_M2) -#define PIPE_LINK_N2(tran) _TRANSCODER(tran, _PIPEA_LINK_N2, _PIPEB_LINK_N2) +#define _PIPEB_DATA_M1 0x61030 +#define _PIPEB_DATA_N1 0x61034 +#define _PIPEB_DATA_M2 0x61038 +#define _PIPEB_DATA_N2 0x6103c +#define _PIPEB_LINK_M1 0x61040 +#define _PIPEB_LINK_N1 0x61044 +#define _PIPEB_LINK_M2 0x61048 +#define _PIPEB_LINK_N2 0x6104c + +#define PIPE_DATA_M1(tran) _TRANSCODER2(tran, _PIPEA_DATA_M1) +#define PIPE_DATA_N1(tran) _TRANSCODER2(tran, _PIPEA_DATA_N1) +#define PIPE_DATA_M2(tran) _TRANSCODER2(tran, _PIPEA_DATA_M2) +#define PIPE_DATA_N2(tran) _TRANSCODER2(tran, _PIPEA_DATA_N2) +#define PIPE_LINK_M1(tran) _TRANSCODER2(tran, _PIPEA_LINK_M1) +#define PIPE_LINK_N1(tran) _TRANSCODER2(tran, _PIPEA_LINK_N1) +#define PIPE_LINK_M2(tran) _TRANSCODER2(tran, _PIPEA_LINK_M2) +#define PIPE_LINK_N2(tran) _TRANSCODER2(tran, _PIPEA_LINK_N2) /* CPU panel fitter */ /* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */ @@ -4025,6 +4517,7 @@ #define GEN8_DE_PIPE_A_IRQ (1<<16) #define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+pipe)) #define GEN8_GT_VECS_IRQ (1<<6) +#define GEN8_GT_PM_IRQ (1<<4) #define GEN8_GT_VCS2_IRQ (1<<3) #define GEN8_GT_VCS1_IRQ (1<<2) #define GEN8_GT_BCS_IRQ (1<<1) @@ -4052,7 +4545,7 @@ #define GEN8_PIPE_SPRITE_FAULT (1 << 9) #define GEN8_PIPE_PRIMARY_FAULT (1 << 8) #define GEN8_PIPE_SPRITE_FLIP_DONE (1 << 5) -#define GEN8_PIPE_FLIP_DONE (1 << 4) +#define GEN8_PIPE_PRIMARY_FLIP_DONE (1 << 4) #define GEN8_PIPE_SCAN_LINE_EVENT (1 << 2) #define GEN8_PIPE_VSYNC (1 << 1) #define GEN8_PIPE_VBLANK (1 << 0) @@ -4084,13 +4577,14 @@ #define ILK_ELPIN_409_SELECT (1 << 25) #define ILK_DPARB_GATE (1<<22) #define ILK_VSDPFD_FULL (1<<21) -#define ILK_DISPLAY_CHICKEN_FUSES 0x42014 -#define ILK_INTERNAL_GRAPHICS_DISABLE (1<<31) -#define ILK_INTERNAL_DISPLAY_DISABLE (1<<30) -#define ILK_DISPLAY_DEBUG_DISABLE (1<<29) -#define ILK_HDCP_DISABLE (1<<25) -#define ILK_eDP_A_DISABLE (1<<24) -#define ILK_DESKTOP (1<<23) +#define FUSE_STRAP 0x42014 +#define ILK_INTERNAL_GRAPHICS_DISABLE (1 << 31) +#define ILK_INTERNAL_DISPLAY_DISABLE (1 << 30) +#define ILK_DISPLAY_DEBUG_DISABLE (1 << 29) +#define ILK_HDCP_DISABLE (1 << 25) +#define ILK_eDP_A_DISABLE (1 << 24) +#define HSW_CDCLK_LIMIT (1 << 24) +#define ILK_DESKTOP (1 << 23) #define ILK_DSPCLK_GATE_D 0x42020 #define ILK_VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) @@ -4109,7 +4603,8 @@ #define _CHICKEN_PIPESL_1_A 0x420b0 #define _CHICKEN_PIPESL_1_B 0x420b4 -#define DPRS_MASK_VBLANK_SRD (1 << 0) +#define HSW_FBCQ_DIS (1 << 22) +#define BDW_DPRS_MASK_VBLANK_SRD (1 << 0) #define CHICKEN_PIPESL_1(pipe) _PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B) #define DISP_ARB_CTL 0x45000 @@ -4120,6 +4615,8 @@ #define GEN7_MSG_CTL 0x45010 #define WAIT_FOR_PCH_RESET_ACK (1<<1) #define WAIT_FOR_PCH_FLR_ACK (1<<0) +#define HSW_NDE_RSTWRN_OPT 0x46408 +#define RESET_PCH_HANDSHAKE_ENABLE (1<<4) /* GEN7 chicken */ #define GEN7_COMMON_SLICE_CHICKEN1 0x7010 @@ -4127,8 +4624,11 @@ #define COMMON_SLICE_CHICKEN2 0x7014 # define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE (1<<0) +#define GEN7_L3SQCREG1 0xB010 +#define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 + #define GEN7_L3CNTLREG1 0xB01C -#define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C4FFF8C +#define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C47FF8C #define GEN7_L3AGDIS (1<<19) #define GEN7_L3_CHICKEN_MODE_REGISTER 0xB030 @@ -4148,9 +4648,6 @@ #define HSW_SCRATCH1 0xb038 #define HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE (1<<27) -#define HSW_FUSE_STRAP 0x42014 -#define HSW_CDCLK_LIMIT (1 << 24) - /* PCH */ /* south display engine interrupt: IBX */ @@ -4436,24 +4933,24 @@ #define HSW_VIDEO_DIP_GCP_B 0x61210 #define HSW_TVIDEO_DIP_CTL(trans) \ - _TRANSCODER(trans, HSW_VIDEO_DIP_CTL_A, HSW_VIDEO_DIP_CTL_B) + _TRANSCODER2(trans, HSW_VIDEO_DIP_CTL_A) #define HSW_TVIDEO_DIP_AVI_DATA(trans) \ - _TRANSCODER(trans, HSW_VIDEO_DIP_AVI_DATA_A, HSW_VIDEO_DIP_AVI_DATA_B) + _TRANSCODER2(trans, HSW_VIDEO_DIP_AVI_DATA_A) #define HSW_TVIDEO_DIP_VS_DATA(trans) \ - _TRANSCODER(trans, HSW_VIDEO_DIP_VS_DATA_A, HSW_VIDEO_DIP_VS_DATA_B) + _TRANSCODER2(trans, HSW_VIDEO_DIP_VS_DATA_A) #define HSW_TVIDEO_DIP_SPD_DATA(trans) \ - _TRANSCODER(trans, HSW_VIDEO_DIP_SPD_DATA_A, HSW_VIDEO_DIP_SPD_DATA_B) + _TRANSCODER2(trans, HSW_VIDEO_DIP_SPD_DATA_A) #define HSW_TVIDEO_DIP_GCP(trans) \ - _TRANSCODER(trans, HSW_VIDEO_DIP_GCP_A, HSW_VIDEO_DIP_GCP_B) + _TRANSCODER2(trans, HSW_VIDEO_DIP_GCP_A) #define HSW_TVIDEO_DIP_VSC_DATA(trans) \ - _TRANSCODER(trans, HSW_VIDEO_DIP_VSC_DATA_A, HSW_VIDEO_DIP_VSC_DATA_B) + _TRANSCODER2(trans, HSW_VIDEO_DIP_VSC_DATA_A) #define HSW_STEREO_3D_CTL_A 0x70020 #define S3D_ENABLE (1<<31) #define HSW_STEREO_3D_CTL_B 0x71020 #define HSW_STEREO_3D_CTL(trans) \ - _TRANSCODER(trans, HSW_STEREO_3D_CTL_A, HSW_STEREO_3D_CTL_A) + _PIPE2(trans, HSW_STEREO_3D_CTL_A) #define _PCH_TRANS_HTOTAL_B 0xe1000 #define _PCH_TRANS_HBLANK_B 0xe1004 @@ -4760,6 +5257,8 @@ #define PORT_TRANS_SEL_CPT(pipe) ((pipe) << 29) #define PORT_TO_PIPE(val) (((val) & (1<<30)) >> 30) #define PORT_TO_PIPE_CPT(val) (((val) & PORT_TRANS_SEL_MASK) >> 29) +#define SDVO_PORT_TO_PIPE_CHV(val) (((val) & (3<<24)) >> 24) +#define DP_PORT_TO_PIPE_CHV(val) (((val) & (3<<16)) >> 16) #define TRANS_DP_CTL_A 0xe0300 #define TRANS_DP_CTL_B 0xe1300 @@ -4816,6 +5315,8 @@ #define EDP_LINK_TRAIN_VOL_EMP_MASK_IVB (0x3f<<22) +#define VLV_PMWGICZ 0x1300a4 + #define FORCEWAKE 0xA18C #define FORCEWAKE_VLV 0x1300b0 #define FORCEWAKE_ACK_VLV 0x1300b4 @@ -4824,15 +5325,22 @@ #define FORCEWAKE_ACK_HSW 0x130044 #define FORCEWAKE_ACK 0x130090 #define VLV_GTLC_WAKE_CTRL 0x130090 +#define VLV_GTLC_RENDER_CTX_EXISTS (1 << 25) +#define VLV_GTLC_MEDIA_CTX_EXISTS (1 << 24) +#define VLV_GTLC_ALLOWWAKEREQ (1 << 0) + #define VLV_GTLC_PW_STATUS 0x130094 -#define VLV_GTLC_PW_RENDER_STATUS_MASK 0x80 -#define VLV_GTLC_PW_MEDIA_STATUS_MASK 0x20 +#define VLV_GTLC_ALLOWWAKEACK (1 << 0) +#define VLV_GTLC_ALLOWWAKEERR (1 << 1) +#define VLV_GTLC_PW_MEDIA_STATUS_MASK (1 << 5) +#define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) #define FORCEWAKE_MT 0xa188 /* multi-threaded */ #define FORCEWAKE_KERNEL 0x1 #define FORCEWAKE_USER 0x2 #define FORCEWAKE_MT_ACK 0x130040 #define ECOBUS 0xa180 #define FORCEWAKE_MT_ENABLE (1<<5) +#define VLV_SPAREG2H 0xA194 #define GTFIFODBG 0x120000 #define GT_FIFO_SBDROPERR (1<<6) @@ -4852,6 +5360,7 @@ #define HSW_EDRAM_PRESENT 0x120010 #define GEN6_UCGCTL1 0x9400 +# define GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE (1 << 16) # define GEN6_BLBUNIT_CLOCK_GATE_DISABLE (1 << 5) # define GEN6_CSUNIT_CLOCK_GATE_DISABLE (1 << 7) @@ -4862,9 +5371,19 @@ # define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE (1 << 12) # define GEN6_RCCUNIT_CLOCK_GATE_DISABLE (1 << 11) +#define GEN6_UCGCTL3 0x9408 + #define GEN7_UCGCTL4 0x940c #define GEN7_L3BANK2X_CLOCK_GATE_DISABLE (1<<25) +#define GEN6_RCGCTL1 0x9410 +#define GEN6_RCGCTL2 0x9414 +#define GEN6_RSTCTL 0x9420 + +#define GEN8_UCGCTL6 0x9430 +#define GEN8_SDEUNIT_CLOCK_GATE_DISABLE (1<<14) + +#define GEN6_GFXPAUSE 0xA000 #define GEN6_RPNSWREQ 0xA008 #define GEN6_TURBO_DISABLE (1<<31) #define GEN6_FREQUENCY(x) ((x)<<25) @@ -4917,6 +5436,9 @@ #define GEN6_RP_UP_EI 0xA068 #define GEN6_RP_DOWN_EI 0xA06C #define GEN6_RP_IDLE_HYSTERSIS 0xA070 +#define GEN6_RPDEUHWTC 0xA080 +#define GEN6_RPDEUC 0xA084 +#define GEN6_RPDEUCSW 0xA088 #define GEN6_RC_STATE 0xA094 #define GEN6_RC1_WAKE_RATE_LIMIT 0xA098 #define GEN6_RC6_WAKE_RATE_LIMIT 0xA09C @@ -4924,11 +5446,15 @@ #define GEN6_RC_EVALUATION_INTERVAL 0xA0A8 #define GEN6_RC_IDLE_HYSTERSIS 0xA0AC #define GEN6_RC_SLEEP 0xA0B0 +#define GEN6_RCUBMABDTMR 0xA0B0 #define GEN6_RC1e_THRESHOLD 0xA0B4 #define GEN6_RC6_THRESHOLD 0xA0B8 #define GEN6_RC6p_THRESHOLD 0xA0BC +#define VLV_RCEDATA 0xA0BC #define GEN6_RC6pp_THRESHOLD 0xA0C0 #define GEN6_PMINTRMSK 0xA168 +#define GEN8_PMINTR_REDIRECT_TO_NON_DISP (1<<31) +#define VLV_PWRDWNUPCTL 0xA294 #define GEN6_PMISR 0x44020 #define GEN6_PMIMR 0x44024 /* rps_lock */ @@ -4945,12 +5471,22 @@ GEN6_PM_RP_DOWN_THRESHOLD | \ GEN6_PM_RP_DOWN_TIMEOUT) +#define GEN7_GT_SCRATCH_BASE 0x4F100 +#define GEN7_GT_SCRATCH_REG_NUM 8 + +#define VLV_GTLC_SURVIVABILITY_REG 0x130098 +#define VLV_GFX_CLK_STATUS_BIT (1<<3) +#define VLV_GFX_CLK_FORCE_ON_BIT (1<<2) + #define GEN6_GT_GFX_RC6_LOCKED 0x138104 #define VLV_COUNTER_CONTROL 0x138104 #define VLV_COUNT_RANGE_HIGH (1<<15) #define VLV_MEDIA_RC6_COUNT_EN (1<<1) #define VLV_RENDER_RC6_COUNT_EN (1<<0) #define GEN6_GT_GFX_RC6 0x138108 +#define VLV_GT_RENDER_RC6 0x138108 +#define VLV_GT_MEDIA_RC6 0x13810C + #define GEN6_GT_GFX_RC6p 0x13810C #define GEN6_GT_GFX_RC6pp 0x138110 @@ -5006,6 +5542,10 @@ #define GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE (1<<10) #define GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE (1<<3) +#define GEN8_ROW_CHICKEN 0xe4f0 +#define PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE (1<<8) +#define STALL_DOP_GATING_DISABLE (1<<5) + #define GEN7_ROW_CHICKEN2 0xe4f4 #define GEN7_ROW_CHICKEN2_GT2 0xf4f4 #define DOP_CLOCK_GATING_DISABLE (1<<0) @@ -5017,7 +5557,7 @@ #define GEN8_CENTROID_PIXEL_OPT_DIS (1<<8) #define GEN8_SAMPLER_POWER_BYPASS_DIS (1<<1) -#define G4X_AUD_VID_DID (dev_priv->info->display_mmio_offset + 0x62020) +#define G4X_AUD_VID_DID (dev_priv->info.display_mmio_offset + 0x62020) #define INTEL_AUDIO_DEVCL 0x808629FB #define INTEL_AUDIO_DEVBLC 0x80862801 #define INTEL_AUDIO_DEVCTG 0x80862802 @@ -5178,8 +5718,8 @@ #define TRANS_DDI_FUNC_CTL_B 0x61400 #define TRANS_DDI_FUNC_CTL_C 0x62400 #define TRANS_DDI_FUNC_CTL_EDP 0x6F400 -#define TRANS_DDI_FUNC_CTL(tran) _TRANSCODER(tran, TRANS_DDI_FUNC_CTL_A, \ - TRANS_DDI_FUNC_CTL_B) +#define TRANS_DDI_FUNC_CTL(tran) _TRANSCODER2(tran, TRANS_DDI_FUNC_CTL_A) + #define TRANS_DDI_FUNC_ENABLE (1<<31) /* Those bits are ignored by pipe EDP since it can only connect to DDI A */ #define TRANS_DDI_PORT_MASK (7<<28) @@ -5311,8 +5851,12 @@ #define SPLL_PLL_ENABLE (1<<31) #define SPLL_PLL_SSC (1<<28) #define SPLL_PLL_NON_SSC (2<<28) +#define SPLL_PLL_LCPLL (3<<28) +#define SPLL_PLL_REF_MASK (3<<28) #define SPLL_PLL_FREQ_810MHz (0<<26) #define SPLL_PLL_FREQ_1350MHz (1<<26) +#define SPLL_PLL_FREQ_2700MHz (2<<26) +#define SPLL_PLL_FREQ_MASK (3<<26) /* WRPLL */ #define WRPLL_CTL1 0x46040 @@ -5323,8 +5867,13 @@ #define WRPLL_PLL_SELECT_LCPLL_2700 (0x03<<28) /* WRPLL divider programming */ #define WRPLL_DIVIDER_REFERENCE(x) ((x)<<0) +#define WRPLL_DIVIDER_REF_MASK (0xff) #define WRPLL_DIVIDER_POST(x) ((x)<<8) +#define WRPLL_DIVIDER_POST_MASK (0x3f<<8) +#define WRPLL_DIVIDER_POST_SHIFT 8 #define WRPLL_DIVIDER_FEEDBACK(x) ((x)<<16) +#define WRPLL_DIVIDER_FB_SHIFT 16 +#define WRPLL_DIVIDER_FB_MASK (0xff<<16) /* Port clock selection */ #define PORT_CLK_SEL_A 0x46100 @@ -5337,6 +5886,7 @@ #define PORT_CLK_SEL_WRPLL1 (4<<29) #define PORT_CLK_SEL_WRPLL2 (5<<29) #define PORT_CLK_SEL_NONE (7<<29) +#define PORT_CLK_SEL_MASK (7<<29) /* Transcoder clock selection */ #define TRANS_CLK_SEL_A 0x46140 @@ -5346,10 +5896,12 @@ #define TRANS_CLK_SEL_DISABLED (0x0<<29) #define TRANS_CLK_SEL_PORT(x) ((x+1)<<29) -#define _TRANSA_MSA_MISC 0x60410 -#define _TRANSB_MSA_MISC 0x61410 -#define TRANS_MSA_MISC(tran) _TRANSCODER(tran, _TRANSA_MSA_MISC, \ - _TRANSB_MSA_MISC) +#define TRANSA_MSA_MISC 0x60410 +#define TRANSB_MSA_MISC 0x61410 +#define TRANSC_MSA_MISC 0x62410 +#define TRANS_EDP_MSA_MISC 0x6f410 +#define TRANS_MSA_MISC(tran) _TRANSCODER2(tran, TRANSA_MSA_MISC) + #define TRANS_MSA_SYNC_CLK (1<<0) #define TRANS_MSA_6_BPC (0<<5) #define TRANS_MSA_8_BPC (1<<5) @@ -5389,6 +5941,8 @@ /* SFUSE_STRAP */ #define SFUSE_STRAP 0xc2014 +#define SFUSE_STRAP_FUSE_LOCK (1<<13) +#define SFUSE_STRAP_DISPLAY_DISABLED (1<<7) #define SFUSE_STRAP_DDIB_DETECTED (1<<2) #define SFUSE_STRAP_DDIC_DETECTED (1<<1) #define SFUSE_STRAP_DDID_DETECTED (1<<0) @@ -5857,4 +6411,12 @@ #define MIPI_READ_DATA_VALID(pipe) _PIPE(pipe, _MIPIA_READ_DATA_VALID, _MIPIB_READ_DATA_VALID) #define READ_DATA_VALID(n) (1 << (n)) +/* For UMS only (deprecated): */ +#define _PALETTE_A (dev_priv->info.display_mmio_offset + 0xa000) +#define _PALETTE_B (dev_priv->info.display_mmio_offset + 0xa800) +#define _DPLL_A (dev_priv->info.display_mmio_offset + 0x6014) +#define _DPLL_B (dev_priv->info.display_mmio_offset + 0x6018) +#define _DPLL_A_MD (dev_priv->info.display_mmio_offset + 0x601c) +#define _DPLL_B_MD (dev_priv->info.display_mmio_offset + 0x6020) + #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 8150fdc08d4..043123c77a1 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -236,19 +236,9 @@ static void i915_save_display(struct drm_device *dev) dev_priv->regfile.savePP_DIVISOR = I915_READ(PP_DIVISOR); } - /* Only regfile.save FBC state on the platform that supports FBC */ - if (HAS_FBC(dev)) { - if (HAS_PCH_SPLIT(dev)) { - dev_priv->regfile.saveDPFC_CB_BASE = I915_READ(ILK_DPFC_CB_BASE); - } else if (IS_GM45(dev)) { - dev_priv->regfile.saveDPFC_CB_BASE = I915_READ(DPFC_CB_BASE); - } else { - dev_priv->regfile.saveFBC_CFB_BASE = I915_READ(FBC_CFB_BASE); - dev_priv->regfile.saveFBC_LL_BASE = I915_READ(FBC_LL_BASE); - dev_priv->regfile.saveFBC_CONTROL2 = I915_READ(FBC_CONTROL2); - dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL); - } - } + /* save FBC interval */ + if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev)) + dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL); if (!drm_core_check_feature(dev, DRIVER_MODESET)) i915_save_vga(dev); @@ -300,18 +290,10 @@ static void i915_restore_display(struct drm_device *dev) /* only restore FBC info on the platform that supports FBC*/ intel_disable_fbc(dev); - if (HAS_FBC(dev)) { - if (HAS_PCH_SPLIT(dev)) { - I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->regfile.saveDPFC_CB_BASE); - } else if (IS_GM45(dev)) { - I915_WRITE(DPFC_CB_BASE, dev_priv->regfile.saveDPFC_CB_BASE); - } else { - I915_WRITE(FBC_CFB_BASE, dev_priv->regfile.saveFBC_CFB_BASE); - I915_WRITE(FBC_LL_BASE, dev_priv->regfile.saveFBC_LL_BASE); - I915_WRITE(FBC_CONTROL2, dev_priv->regfile.saveFBC_CONTROL2); - I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL); - } - } + + /* restore FBC interval */ + if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev)) + I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL); if (!drm_core_check_feature(dev, DRIVER_MODESET)) i915_restore_vga(dev); @@ -324,10 +306,6 @@ int i915_save_state(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int i; - if (INTEL_INFO(dev)->gen <= 4) - pci_read_config_byte(dev->pdev, LBB, - &dev_priv->regfile.saveLBB); - mutex_lock(&dev->struct_mutex); i915_save_display(dev); @@ -350,8 +328,6 @@ int i915_save_state(struct drm_device *dev) } } - intel_disable_gt_powersave(dev); - /* Cache mode state */ if (INTEL_INFO(dev)->gen < 7) dev_priv->regfile.saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); @@ -377,10 +353,6 @@ int i915_restore_state(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int i; - if (INTEL_INFO(dev)->gen <= 4) - pci_write_config_byte(dev->pdev, LBB, - dev_priv->regfile.saveLBB); - mutex_lock(&dev->struct_mutex); i915_gem_restore_fences(dev); diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 33bcae314bf..86ce39aad0f 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -186,7 +186,7 @@ i915_l3_write(struct file *filp, struct kobject *kobj, struct drm_minor *dminor = dev_to_drm_minor(dev); struct drm_device *drm_dev = dminor->dev; struct drm_i915_private *dev_priv = drm_dev->dev_private; - struct i915_hw_context *ctx; + struct intel_context *ctx; u32 *temp = NULL; /* Just here to make handling failures easy */ int slice = (int)(uintptr_t)attr->private; int ret; @@ -263,16 +263,20 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev, flush_delayed_work(&dev_priv->rps.delayed_resume_work); + intel_runtime_pm_get(dev_priv); + mutex_lock(&dev_priv->rps.hw_lock); if (IS_VALLEYVIEW(dev_priv->dev)) { u32 freq; freq = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); ret = vlv_gpu_freq(dev_priv, (freq >> 8) & 0xff); } else { - ret = dev_priv->rps.cur_delay * GT_FREQUENCY_MULTIPLIER; + ret = dev_priv->rps.cur_freq * GT_FREQUENCY_MULTIPLIER; } mutex_unlock(&dev_priv->rps.hw_lock); + intel_runtime_pm_put(dev_priv); + return snprintf(buf, PAGE_SIZE, "%d\n", ret); } @@ -284,7 +288,7 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev, struct drm_i915_private *dev_priv = dev->dev_private; return snprintf(buf, PAGE_SIZE, "%d\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay)); + vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq)); } static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf) @@ -298,9 +302,9 @@ static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute mutex_lock(&dev_priv->rps.hw_lock); if (IS_VALLEYVIEW(dev_priv->dev)) - ret = vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay); + ret = vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit); else - ret = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER; + ret = dev_priv->rps.max_freq_softlimit * GT_FREQUENCY_MULTIPLIER; mutex_unlock(&dev_priv->rps.hw_lock); return snprintf(buf, PAGE_SIZE, "%d\n", ret); @@ -313,7 +317,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 val, rp_state_cap, hw_max, hw_min, non_oc_max; + u32 val; ssize_t ret; ret = kstrtou32(buf, 0, &val); @@ -324,38 +328,34 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, mutex_lock(&dev_priv->rps.hw_lock); - if (IS_VALLEYVIEW(dev_priv->dev)) { + if (IS_VALLEYVIEW(dev_priv->dev)) val = vlv_freq_opcode(dev_priv, val); - - hw_max = valleyview_rps_max_freq(dev_priv); - hw_min = valleyview_rps_min_freq(dev_priv); - non_oc_max = hw_max; - } else { + else val /= GT_FREQUENCY_MULTIPLIER; - rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); - hw_max = dev_priv->rps.hw_max; - non_oc_max = (rp_state_cap & 0xff); - hw_min = ((rp_state_cap & 0xff0000) >> 16); - } - - if (val < hw_min || val > hw_max || - val < dev_priv->rps.min_delay) { + if (val < dev_priv->rps.min_freq || + val > dev_priv->rps.max_freq || + val < dev_priv->rps.min_freq_softlimit) { mutex_unlock(&dev_priv->rps.hw_lock); return -EINVAL; } - if (val > non_oc_max) + if (val > dev_priv->rps.rp0_freq) DRM_DEBUG("User requested overclocking to %d\n", val * GT_FREQUENCY_MULTIPLIER); - dev_priv->rps.max_delay = val; + dev_priv->rps.max_freq_softlimit = val; - if (dev_priv->rps.cur_delay > val) { + if (dev_priv->rps.cur_freq > val) { if (IS_VALLEYVIEW(dev)) valleyview_set_rps(dev, val); else gen6_set_rps(dev, val); + } else if (!IS_VALLEYVIEW(dev)) { + /* We still need gen6_set_rps to process the new max_delay and + * update the interrupt limits even though frequency request is + * unchanged. */ + gen6_set_rps(dev, dev_priv->rps.cur_freq); } mutex_unlock(&dev_priv->rps.hw_lock); @@ -374,9 +374,9 @@ static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute mutex_lock(&dev_priv->rps.hw_lock); if (IS_VALLEYVIEW(dev_priv->dev)) - ret = vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay); + ret = vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit); else - ret = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER; + ret = dev_priv->rps.min_freq_softlimit * GT_FREQUENCY_MULTIPLIER; mutex_unlock(&dev_priv->rps.hw_lock); return snprintf(buf, PAGE_SIZE, "%d\n", ret); @@ -389,7 +389,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 val, rp_state_cap, hw_max, hw_min; + u32 val; ssize_t ret; ret = kstrtou32(buf, 0, &val); @@ -400,31 +400,30 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, mutex_lock(&dev_priv->rps.hw_lock); - if (IS_VALLEYVIEW(dev)) { + if (IS_VALLEYVIEW(dev)) val = vlv_freq_opcode(dev_priv, val); - - hw_max = valleyview_rps_max_freq(dev_priv); - hw_min = valleyview_rps_min_freq(dev_priv); - } else { + else val /= GT_FREQUENCY_MULTIPLIER; - rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); - hw_max = dev_priv->rps.hw_max; - hw_min = ((rp_state_cap & 0xff0000) >> 16); - } - - if (val < hw_min || val > hw_max || val > dev_priv->rps.max_delay) { + if (val < dev_priv->rps.min_freq || + val > dev_priv->rps.max_freq || + val > dev_priv->rps.max_freq_softlimit) { mutex_unlock(&dev_priv->rps.hw_lock); return -EINVAL; } - dev_priv->rps.min_delay = val; + dev_priv->rps.min_freq_softlimit = val; - if (dev_priv->rps.cur_delay < val) { + if (dev_priv->rps.cur_freq < val) { if (IS_VALLEYVIEW(dev)) valleyview_set_rps(dev, val); else gen6_set_rps(dev, val); + } else if (!IS_VALLEYVIEW(dev)) { + /* We still need gen6_set_rps to process the new min_delay and + * update the interrupt limits even though frequency request is + * unchanged. */ + gen6_set_rps(dev, dev_priv->rps.cur_freq); } mutex_unlock(&dev_priv->rps.hw_lock); diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 6e580c98ded..f5aa0067755 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -7,6 +7,7 @@ #include <drm/drmP.h> #include "i915_drv.h" +#include "intel_drv.h" #include "intel_ringbuffer.h" #undef TRACE_SYSTEM @@ -14,6 +15,80 @@ #define TRACE_SYSTEM_STRING __stringify(TRACE_SYSTEM) #define TRACE_INCLUDE_FILE i915_trace +/* pipe updates */ + +TRACE_EVENT(i915_pipe_update_start, + TP_PROTO(struct intel_crtc *crtc, u32 min, u32 max), + TP_ARGS(crtc, min, max), + + TP_STRUCT__entry( + __field(enum pipe, pipe) + __field(u32, frame) + __field(u32, scanline) + __field(u32, min) + __field(u32, max) + ), + + TP_fast_assign( + __entry->pipe = crtc->pipe; + __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev, + crtc->pipe); + __entry->scanline = intel_get_crtc_scanline(crtc); + __entry->min = min; + __entry->max = max; + ), + + TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u", + pipe_name(__entry->pipe), __entry->frame, + __entry->scanline, __entry->min, __entry->max) +); + +TRACE_EVENT(i915_pipe_update_vblank_evaded, + TP_PROTO(struct intel_crtc *crtc, u32 min, u32 max, u32 frame), + TP_ARGS(crtc, min, max, frame), + + TP_STRUCT__entry( + __field(enum pipe, pipe) + __field(u32, frame) + __field(u32, scanline) + __field(u32, min) + __field(u32, max) + ), + + TP_fast_assign( + __entry->pipe = crtc->pipe; + __entry->frame = frame; + __entry->scanline = intel_get_crtc_scanline(crtc); + __entry->min = min; + __entry->max = max; + ), + + TP_printk("pipe %c, frame=%u, scanline=%u, min=%u, max=%u", + pipe_name(__entry->pipe), __entry->frame, + __entry->scanline, __entry->min, __entry->max) +); + +TRACE_EVENT(i915_pipe_update_end, + TP_PROTO(struct intel_crtc *crtc, u32 frame), + TP_ARGS(crtc, frame), + + TP_STRUCT__entry( + __field(enum pipe, pipe) + __field(u32, frame) + __field(u32, scanline) + ), + + TP_fast_assign( + __entry->pipe = crtc->pipe; + __entry->frame = frame; + __entry->scanline = intel_get_crtc_scanline(crtc); + ), + + TP_printk("pipe %c, frame=%u, scanline=%u", + pipe_name(__entry->pipe), __entry->frame, + __entry->scanline) +); + /* object tracking */ TRACE_EVENT(i915_gem_object_create, @@ -34,15 +109,15 @@ TRACE_EVENT(i915_gem_object_create, ); TRACE_EVENT(i915_vma_bind, - TP_PROTO(struct i915_vma *vma, bool mappable), - TP_ARGS(vma, mappable), + TP_PROTO(struct i915_vma *vma, unsigned flags), + TP_ARGS(vma, flags), TP_STRUCT__entry( __field(struct drm_i915_gem_object *, obj) __field(struct i915_address_space *, vm) __field(u32, offset) __field(u32, size) - __field(bool, mappable) + __field(unsigned, flags) ), TP_fast_assign( @@ -50,12 +125,12 @@ TRACE_EVENT(i915_vma_bind, __entry->vm = vma->vm; __entry->offset = vma->node.start; __entry->size = vma->node.size; - __entry->mappable = mappable; + __entry->flags = flags; ), TP_printk("obj=%p, offset=%08x size=%x%s vm=%p", __entry->obj, __entry->offset, __entry->size, - __entry->mappable ? ", mappable" : "", + __entry->flags & PIN_MAPPABLE ? ", mappable" : "", __entry->vm) ); @@ -196,26 +271,26 @@ DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy, ); TRACE_EVENT(i915_gem_evict, - TP_PROTO(struct drm_device *dev, u32 size, u32 align, bool mappable), - TP_ARGS(dev, size, align, mappable), + TP_PROTO(struct drm_device *dev, u32 size, u32 align, unsigned flags), + TP_ARGS(dev, size, align, flags), TP_STRUCT__entry( __field(u32, dev) __field(u32, size) __field(u32, align) - __field(bool, mappable) + __field(unsigned, flags) ), TP_fast_assign( __entry->dev = dev->primary->index; __entry->size = size; __entry->align = align; - __entry->mappable = mappable; + __entry->flags = flags; ), TP_printk("dev=%d, size=%d, align=%d %s", __entry->dev, __entry->size, __entry->align, - __entry->mappable ? ", mappable" : "") + __entry->flags & PIN_MAPPABLE ? ", mappable" : "") ); TRACE_EVENT(i915_gem_evict_everything, @@ -238,19 +313,21 @@ TRACE_EVENT(i915_gem_evict_vm, TP_ARGS(vm), TP_STRUCT__entry( + __field(u32, dev) __field(struct i915_address_space *, vm) ), TP_fast_assign( + __entry->dev = vm->dev->primary->index; __entry->vm = vm; ), - TP_printk("dev=%d, vm=%p", __entry->vm->dev->primary->index, __entry->vm) + TP_printk("dev=%d, vm=%p", __entry->dev, __entry->vm) ); TRACE_EVENT(i915_gem_ring_sync_to, - TP_PROTO(struct intel_ring_buffer *from, - struct intel_ring_buffer *to, + TP_PROTO(struct intel_engine_cs *from, + struct intel_engine_cs *to, u32 seqno), TP_ARGS(from, to, seqno), @@ -275,7 +352,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, ); TRACE_EVENT(i915_gem_ring_dispatch, - TP_PROTO(struct intel_ring_buffer *ring, u32 seqno, u32 flags), + TP_PROTO(struct intel_engine_cs *ring, u32 seqno, u32 flags), TP_ARGS(ring, seqno, flags), TP_STRUCT__entry( @@ -298,7 +375,7 @@ TRACE_EVENT(i915_gem_ring_dispatch, ); TRACE_EVENT(i915_gem_ring_flush, - TP_PROTO(struct intel_ring_buffer *ring, u32 invalidate, u32 flush), + TP_PROTO(struct intel_engine_cs *ring, u32 invalidate, u32 flush), TP_ARGS(ring, invalidate, flush), TP_STRUCT__entry( @@ -321,7 +398,7 @@ TRACE_EVENT(i915_gem_ring_flush, ); DECLARE_EVENT_CLASS(i915_gem_request, - TP_PROTO(struct intel_ring_buffer *ring, u32 seqno), + TP_PROTO(struct intel_engine_cs *ring, u32 seqno), TP_ARGS(ring, seqno), TP_STRUCT__entry( @@ -341,12 +418,12 @@ DECLARE_EVENT_CLASS(i915_gem_request, ); DEFINE_EVENT(i915_gem_request, i915_gem_request_add, - TP_PROTO(struct intel_ring_buffer *ring, u32 seqno), + TP_PROTO(struct intel_engine_cs *ring, u32 seqno), TP_ARGS(ring, seqno) ); TRACE_EVENT(i915_gem_request_complete, - TP_PROTO(struct intel_ring_buffer *ring), + TP_PROTO(struct intel_engine_cs *ring), TP_ARGS(ring), TP_STRUCT__entry( @@ -366,12 +443,12 @@ TRACE_EVENT(i915_gem_request_complete, ); DEFINE_EVENT(i915_gem_request, i915_gem_request_retire, - TP_PROTO(struct intel_ring_buffer *ring, u32 seqno), + TP_PROTO(struct intel_engine_cs *ring, u32 seqno), TP_ARGS(ring, seqno) ); TRACE_EVENT(i915_gem_request_wait_begin, - TP_PROTO(struct intel_ring_buffer *ring, u32 seqno), + TP_PROTO(struct intel_engine_cs *ring, u32 seqno), TP_ARGS(ring, seqno), TP_STRUCT__entry( @@ -400,12 +477,12 @@ TRACE_EVENT(i915_gem_request_wait_begin, ); DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_end, - TP_PROTO(struct intel_ring_buffer *ring, u32 seqno), + TP_PROTO(struct intel_engine_cs *ring, u32 seqno), TP_ARGS(ring, seqno) ); DECLARE_EVENT_CLASS(i915_ring, - TP_PROTO(struct intel_ring_buffer *ring), + TP_PROTO(struct intel_engine_cs *ring), TP_ARGS(ring), TP_STRUCT__entry( @@ -422,12 +499,12 @@ DECLARE_EVENT_CLASS(i915_ring, ); DEFINE_EVENT(i915_ring, i915_ring_wait_begin, - TP_PROTO(struct intel_ring_buffer *ring), + TP_PROTO(struct intel_engine_cs *ring), TP_ARGS(ring) ); DEFINE_EVENT(i915_ring, i915_ring_wait_end, - TP_PROTO(struct intel_ring_buffer *ring), + TP_PROTO(struct intel_engine_cs *ring), TP_ARGS(ring) ); diff --git a/drivers/gpu/drm/i915/i915_ums.c b/drivers/gpu/drm/i915/i915_ums.c index caa18e85581..480da593e6c 100644 --- a/drivers/gpu/drm/i915/i915_ums.c +++ b/drivers/gpu/drm/i915/i915_ums.c @@ -271,6 +271,10 @@ void i915_save_display_reg(struct drm_device *dev) /* FIXME: regfile.save TV & SDVO state */ /* Backlight */ + if (INTEL_INFO(dev)->gen <= 4) + pci_read_config_byte(dev->pdev, PCI_LBPC, + &dev_priv->regfile.saveLBB); + if (HAS_PCH_SPLIT(dev)) { dev_priv->regfile.saveBLC_PWM_CTL = I915_READ(BLC_PWM_PCH_CTL1); dev_priv->regfile.saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_PCH_CTL2); @@ -293,6 +297,10 @@ void i915_restore_display_reg(struct drm_device *dev) int i; /* Backlight */ + if (INTEL_INFO(dev)->gen <= 4) + pci_write_config_byte(dev->pdev, PCI_LBPC, + dev_priv->regfile.saveLBB); + if (HAS_PCH_SPLIT(dev)) { I915_WRITE(BLC_PWM_PCH_CTL1, dev_priv->regfile.saveBLC_PWM_CTL); I915_WRITE(BLC_PWM_PCH_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2); diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index f22041973f3..827498e081d 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -49,13 +49,19 @@ find_section(struct bdb_header *bdb, int section_id) total = bdb->bdb_size; /* walk the sections looking for section_id */ - while (index < total) { + while (index + 3 < total) { current_id = *(base + index); index++; + current_size = *((u16 *)(base + index)); index += 2; + + if (index + current_size > total) + return NULL; + if (current_id == section_id) return base + index; + index += current_size; } @@ -206,7 +212,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv, const struct lvds_dvo_timing *panel_dvo_timing; const struct lvds_fp_timing *fp_timing; struct drm_display_mode *panel_fixed_mode; - int i, downclock; + int i, downclock, drrs_mode; lvds_options = find_section(bdb, BDB_LVDS_OPTIONS); if (!lvds_options) @@ -218,6 +224,28 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv, panel_type = lvds_options->panel_type; + drrs_mode = (lvds_options->dps_panel_type_bits + >> (panel_type * 2)) & MODE_MASK; + /* + * VBT has static DRRS = 0 and seamless DRRS = 2. + * The below piece of code is required to adjust vbt.drrs_type + * to match the enum drrs_support_type. + */ + switch (drrs_mode) { + case 0: + dev_priv->vbt.drrs_type = STATIC_DRRS_SUPPORT; + DRM_DEBUG_KMS("DRRS supported mode is static\n"); + break; + case 2: + dev_priv->vbt.drrs_type = SEAMLESS_DRRS_SUPPORT; + DRM_DEBUG_KMS("DRRS supported mode is seamless\n"); + break; + default: + dev_priv->vbt.drrs_type = DRRS_NOT_SUPPORTED; + DRM_DEBUG_KMS("DRRS not supported (VBT input)\n"); + break; + } + lvds_lfp_data = find_section(bdb, BDB_LVDS_LFP_DATA); if (!lvds_lfp_data) return; @@ -259,7 +287,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv, downclock = dvo_timing->clock; } - if (downclock < panel_dvo_timing->clock && i915_lvds_downclock) { + if (downclock < panel_dvo_timing->clock && i915.lvds_downclock) { dev_priv->lvds_downclock_avail = 1; dev_priv->lvds_downclock = downclock * 10; DRM_DEBUG_KMS("LVDS downclock is found in VBT. " @@ -299,6 +327,13 @@ parse_lfp_backlight(struct drm_i915_private *dev_priv, struct bdb_header *bdb) entry = &backlight_data->data[panel_type]; + dev_priv->vbt.backlight.present = entry->type == BDB_BACKLIGHT_TYPE_PWM; + if (!dev_priv->vbt.backlight.present) { + DRM_DEBUG_KMS("PWM backlight not present in VBT (type %u)\n", + entry->type); + return; + } + dev_priv->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz; dev_priv->vbt.backlight.active_low_pwm = entry->active_low_pwm; DRM_DEBUG_KMS("VBT backlight PWM modulation frequency %u Hz, " @@ -318,7 +353,7 @@ parse_sdvo_panel_data(struct drm_i915_private *dev_priv, struct drm_display_mode *panel_fixed_mode; int index; - index = i915_vbt_sdvo_panel_type; + index = i915.vbt_sdvo_panel_type; if (index == -2) { DRM_DEBUG_KMS("Ignore SDVO panel mode from BIOS VBT tables.\n"); return; @@ -516,6 +551,16 @@ parse_driver_features(struct drm_i915_private *dev_priv, if (driver->dual_frequency) dev_priv->render_reclock_avail = true; + + DRM_DEBUG_KMS("DRRS State Enabled:%d\n", driver->drrs_enabled); + /* + * If DRRS is not supported, drrs_type has to be set to 0. + * This is because, VBT is configured in such a way that + * static DRRS is 0 and DRRS not supported is represented by + * driver->drrs_enabled=false + */ + if (!driver->drrs_enabled) + dev_priv->vbt.drrs_type = DRRS_NOT_SUPPORTED; } static void @@ -550,63 +595,289 @@ parse_edp(struct drm_i915_private *dev_priv, struct bdb_header *bdb) dev_priv->vbt.edp_pps = *edp_pps; - dev_priv->vbt.edp_rate = edp_link_params->rate ? DP_LINK_BW_2_7 : - DP_LINK_BW_1_62; + switch (edp_link_params->rate) { + case EDP_RATE_1_62: + dev_priv->vbt.edp_rate = DP_LINK_BW_1_62; + break; + case EDP_RATE_2_7: + dev_priv->vbt.edp_rate = DP_LINK_BW_2_7; + break; + default: + DRM_DEBUG_KMS("VBT has unknown eDP link rate value %u\n", + edp_link_params->rate); + break; + } + switch (edp_link_params->lanes) { - case 0: + case EDP_LANE_1: dev_priv->vbt.edp_lanes = 1; break; - case 1: + case EDP_LANE_2: dev_priv->vbt.edp_lanes = 2; break; - case 3: - default: + case EDP_LANE_4: dev_priv->vbt.edp_lanes = 4; break; + default: + DRM_DEBUG_KMS("VBT has unknown eDP lane count value %u\n", + edp_link_params->lanes); + break; } + switch (edp_link_params->preemphasis) { - case 0: + case EDP_PREEMPHASIS_NONE: dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_0; break; - case 1: + case EDP_PREEMPHASIS_3_5dB: dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_3_5; break; - case 2: + case EDP_PREEMPHASIS_6dB: dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_6; break; - case 3: + case EDP_PREEMPHASIS_9_5dB: dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_9_5; break; + default: + DRM_DEBUG_KMS("VBT has unknown eDP pre-emphasis value %u\n", + edp_link_params->preemphasis); + break; } + switch (edp_link_params->vswing) { - case 0: + case EDP_VSWING_0_4V: dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_400; break; - case 1: + case EDP_VSWING_0_6V: dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_600; break; - case 2: + case EDP_VSWING_0_8V: dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_800; break; - case 3: + case EDP_VSWING_1_2V: dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_1200; break; + default: + DRM_DEBUG_KMS("VBT has unknown eDP voltage swing value %u\n", + edp_link_params->vswing); + break; } } +static u8 *goto_next_sequence(u8 *data, int *size) +{ + u16 len; + int tmp = *size; + + if (--tmp < 0) + return NULL; + + /* goto first element */ + data++; + while (1) { + switch (*data) { + case MIPI_SEQ_ELEM_SEND_PKT: + /* + * skip by this element payload size + * skip elem id, command flag and data type + */ + tmp -= 5; + if (tmp < 0) + return NULL; + + data += 3; + len = *((u16 *)data); + + tmp -= len; + if (tmp < 0) + return NULL; + + /* skip by len */ + data = data + 2 + len; + break; + case MIPI_SEQ_ELEM_DELAY: + /* skip by elem id, and delay is 4 bytes */ + tmp -= 5; + if (tmp < 0) + return NULL; + + data += 5; + break; + case MIPI_SEQ_ELEM_GPIO: + tmp -= 3; + if (tmp < 0) + return NULL; + + data += 3; + break; + default: + DRM_ERROR("Unknown element\n"); + return NULL; + } + + /* end of sequence ? */ + if (*data == 0) + break; + } + + /* goto next sequence or end of block byte */ + if (--tmp < 0) + return NULL; + + data++; + + /* update amount of data left for the sequence block to be parsed */ + *size = tmp; + return data; +} + static void parse_mipi(struct drm_i915_private *dev_priv, struct bdb_header *bdb) { - struct bdb_mipi *mipi; + struct bdb_mipi_config *start; + struct bdb_mipi_sequence *sequence; + struct mipi_config *config; + struct mipi_pps_data *pps; + u8 *data, *seq_data; + int i, panel_id, seq_size; + u16 block_size; + + /* parse MIPI blocks only if LFP type is MIPI */ + if (!dev_priv->vbt.has_mipi) + return; - mipi = find_section(bdb, BDB_MIPI); - if (!mipi) { - DRM_DEBUG_KMS("No MIPI BDB found"); + /* Initialize this to undefined indicating no generic MIPI support */ + dev_priv->vbt.dsi.panel_id = MIPI_DSI_UNDEFINED_PANEL_ID; + + /* Block #40 is already parsed and panel_fixed_mode is + * stored in dev_priv->lfp_lvds_vbt_mode + * resuse this when needed + */ + + /* Parse #52 for panel index used from panel_type already + * parsed + */ + start = find_section(bdb, BDB_MIPI_CONFIG); + if (!start) { + DRM_DEBUG_KMS("No MIPI config BDB found"); return; } - /* XXX: add more info */ - dev_priv->vbt.dsi.panel_id = mipi->panel_id; + DRM_DEBUG_DRIVER("Found MIPI Config block, panel index = %d\n", + panel_type); + + /* + * get hold of the correct configuration block and pps data as per + * the panel_type as index + */ + config = &start->config[panel_type]; + pps = &start->pps[panel_type]; + + /* store as of now full data. Trim when we realise all is not needed */ + dev_priv->vbt.dsi.config = kmemdup(config, sizeof(struct mipi_config), GFP_KERNEL); + if (!dev_priv->vbt.dsi.config) + return; + + dev_priv->vbt.dsi.pps = kmemdup(pps, sizeof(struct mipi_pps_data), GFP_KERNEL); + if (!dev_priv->vbt.dsi.pps) { + kfree(dev_priv->vbt.dsi.config); + return; + } + + /* We have mandatory mipi config blocks. Initialize as generic panel */ + dev_priv->vbt.dsi.panel_id = MIPI_DSI_GENERIC_PANEL_ID; + + /* Check if we have sequence block as well */ + sequence = find_section(bdb, BDB_MIPI_SEQUENCE); + if (!sequence) { + DRM_DEBUG_KMS("No MIPI Sequence found, parsing complete\n"); + return; + } + + DRM_DEBUG_DRIVER("Found MIPI sequence block\n"); + + block_size = get_blocksize(sequence); + + /* + * parse the sequence block for individual sequences + */ + dev_priv->vbt.dsi.seq_version = sequence->version; + + seq_data = &sequence->data[0]; + + /* + * sequence block is variable length and hence we need to parse and + * get the sequence data for specific panel id + */ + for (i = 0; i < MAX_MIPI_CONFIGURATIONS; i++) { + panel_id = *seq_data; + seq_size = *((u16 *) (seq_data + 1)); + if (panel_id == panel_type) + break; + + /* skip the sequence including seq header of 3 bytes */ + seq_data = seq_data + 3 + seq_size; + if ((seq_data - &sequence->data[0]) > block_size) { + DRM_ERROR("Sequence start is beyond sequence block size, corrupted sequence block\n"); + return; + } + } + + if (i == MAX_MIPI_CONFIGURATIONS) { + DRM_ERROR("Sequence block detected but no valid configuration\n"); + return; + } + + /* check if found sequence is completely within the sequence block + * just being paranoid */ + if (seq_size > block_size) { + DRM_ERROR("Corrupted sequence/size, bailing out\n"); + return; + } + + /* skip the panel id(1 byte) and seq size(2 bytes) */ + dev_priv->vbt.dsi.data = kmemdup(seq_data + 3, seq_size, GFP_KERNEL); + if (!dev_priv->vbt.dsi.data) + return; + + /* + * loop into the sequence data and split into multiple sequneces + * There are only 5 types of sequences as of now + */ + data = dev_priv->vbt.dsi.data; + dev_priv->vbt.dsi.size = seq_size; + + /* two consecutive 0x00 indicate end of all sequences */ + while (1) { + int seq_id = *data; + if (MIPI_SEQ_MAX > seq_id && seq_id > MIPI_SEQ_UNDEFINED) { + dev_priv->vbt.dsi.sequence[seq_id] = data; + DRM_DEBUG_DRIVER("Found mipi sequence - %d\n", seq_id); + } else { + DRM_ERROR("undefined sequence\n"); + goto err; + } + + /* partial parsing to skip elements */ + data = goto_next_sequence(data, &seq_size); + + if (data == NULL) { + DRM_ERROR("Sequence elements going beyond block itself. Sequence block parsing failed\n"); + goto err; + } + + if (*data == 0) + break; /* end of sequence reached */ + } + + DRM_DEBUG_DRIVER("MIPI related vbt parsing complete\n"); + return; +err: + kfree(dev_priv->vbt.dsi.data); + dev_priv->vbt.dsi.data = NULL; + + /* error during parsing so set all pointers to null + * because of partial parsing */ + memset(dev_priv->vbt.dsi.sequence, 0, MIPI_SEQ_MAX); } static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port, @@ -789,6 +1060,15 @@ parse_device_mapping(struct drm_i915_private *dev_priv, /* skip the device block if device type is invalid */ continue; } + + if (p_child->common.dvo_port >= DVO_PORT_MIPIA + && p_child->common.dvo_port <= DVO_PORT_MIPID + &&p_child->common.device_type & DEVICE_TYPE_MIPI_OUTPUT) { + DRM_DEBUG_KMS("Found MIPI as LFP\n"); + dev_priv->vbt.has_mipi = 1; + dev_priv->vbt.dsi.port = p_child->common.dvo_port; + } + child_dev_ptr = dev_priv->vbt.child_dev + count; count++; memcpy((void *)child_dev_ptr, (void *)p_child, @@ -805,6 +1085,9 @@ init_vbt_defaults(struct drm_i915_private *dev_priv) dev_priv->vbt.crt_ddc_pin = GMBUS_PORT_VGADDC; + /* Default to having backlight */ + dev_priv->vbt.backlight.present = true; + /* LFP panel data */ dev_priv->vbt.lvds_dither = 1; dev_priv->vbt.lvds_vbt = 0; @@ -859,6 +1142,46 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = { { } }; +static struct bdb_header *validate_vbt(char *base, size_t size, + struct vbt_header *vbt, + const char *source) +{ + size_t offset; + struct bdb_header *bdb; + + if (vbt == NULL) { + DRM_DEBUG_DRIVER("VBT signature missing\n"); + return NULL; + } + + offset = (char *)vbt - base; + if (offset + sizeof(struct vbt_header) > size) { + DRM_DEBUG_DRIVER("VBT header incomplete\n"); + return NULL; + } + + if (memcmp(vbt->signature, "$VBT", 4)) { + DRM_DEBUG_DRIVER("VBT invalid signature\n"); + return NULL; + } + + offset += vbt->bdb_offset; + if (offset + sizeof(struct bdb_header) > size) { + DRM_DEBUG_DRIVER("BDB header incomplete\n"); + return NULL; + } + + bdb = (struct bdb_header *)(base + offset); + if (offset + bdb->bdb_size > size) { + DRM_DEBUG_DRIVER("BDB incomplete\n"); + return NULL; + } + + DRM_DEBUG_KMS("Using VBT from %s: %20s\n", + source, vbt->signature); + return bdb; +} + /** * intel_parse_bios - find VBT and initialize settings from the BIOS * @dev: DRM device @@ -882,20 +1205,13 @@ intel_parse_bios(struct drm_device *dev) init_vbt_defaults(dev_priv); /* XXX Should this validation be moved to intel_opregion.c? */ - if (!dmi_check_system(intel_no_opregion_vbt) && dev_priv->opregion.vbt) { - struct vbt_header *vbt = dev_priv->opregion.vbt; - if (memcmp(vbt->signature, "$VBT", 4) == 0) { - DRM_DEBUG_KMS("Using VBT from OpRegion: %20s\n", - vbt->signature); - bdb = (struct bdb_header *)((char *)vbt + vbt->bdb_offset); - } else - dev_priv->opregion.vbt = NULL; - } + if (!dmi_check_system(intel_no_opregion_vbt) && dev_priv->opregion.vbt) + bdb = validate_vbt((char *)dev_priv->opregion.header, OPREGION_SIZE, + (struct vbt_header *)dev_priv->opregion.vbt, + "OpRegion"); if (bdb == NULL) { - struct vbt_header *vbt = NULL; - size_t size; - int i; + size_t i, size; bios = pci_map_rom(pdev, &size); if (!bios) @@ -903,19 +1219,18 @@ intel_parse_bios(struct drm_device *dev) /* Scour memory looking for the VBT signature */ for (i = 0; i + 4 < size; i++) { - if (!memcmp(bios + i, "$VBT", 4)) { - vbt = (struct vbt_header *)(bios + i); + if (memcmp(bios + i, "$VBT", 4) == 0) { + bdb = validate_vbt(bios, size, + (struct vbt_header *)(bios + i), + "PCI ROM"); break; } } - if (!vbt) { - DRM_DEBUG_DRIVER("VBT signature missing\n"); + if (!bdb) { pci_unmap_rom(pdev, bios); return -1; } - - bdb = (struct bdb_header *)(bios + i + vbt->bdb_offset); } /* Grab useful general definitions */ diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h index 282de5e9f39..b9866779633 100644 --- a/drivers/gpu/drm/i915/intel_bios.h +++ b/drivers/gpu/drm/i915/intel_bios.h @@ -104,7 +104,8 @@ struct vbios_data { #define BDB_LVDS_LFP_DATA 42 #define BDB_LVDS_BACKLIGHT 43 #define BDB_LVDS_POWER 44 -#define BDB_MIPI 50 +#define BDB_MIPI_CONFIG 52 +#define BDB_MIPI_SEQUENCE 53 #define BDB_SKIP 254 /* VBIOS private block, ignore */ struct bdb_general_features { @@ -281,6 +282,9 @@ struct bdb_general_definitions { union child_device_config devices[0]; } __packed; +/* Mask for DRRS / Panel Channel / SSC / BLT control bits extraction */ +#define MODE_MASK 0x3 + struct bdb_lvds_options { u8 panel_type; u8 rsvd1; @@ -293,6 +297,18 @@ struct bdb_lvds_options { u8 lvds_edid:1; u8 rsvd2:1; u8 rsvd4; + /* LVDS Panel channel bits stored here */ + u32 lvds_panel_channel_bits; + /* LVDS SSC (Spread Spectrum Clock) bits stored here. */ + u16 ssc_bits; + u16 ssc_freq; + u16 ssc_ddt; + /* Panel color depth defined here */ + u16 panel_color_depth; + /* LVDS panel type bits stored here */ + u32 dps_panel_type_bits; + /* LVDS backlight control type bits stored here */ + u32 blt_control_type_bits; } __packed; /* LFP pointer table contains entries to the struct below */ @@ -373,6 +389,9 @@ struct bdb_lvds_lfp_data { struct bdb_lvds_lfp_data_entry data[16]; } __packed; +#define BDB_BACKLIGHT_TYPE_NONE 0 +#define BDB_BACKLIGHT_TYPE_PWM 2 + struct bdb_lfp_backlight_data_entry { u8 type:2; u8 active_low_pwm:1; @@ -478,6 +497,20 @@ struct bdb_driver_features { u8 hdmi_termination; u8 custom_vbt_version; + /* Driver features data block */ + u16 rmpm_enabled:1; + u16 s2ddt_enabled:1; + u16 dpst_enabled:1; + u16 bltclt_enabled:1; + u16 adb_enabled:1; + u16 drrs_enabled:1; + u16 grs_enabled:1; + u16 gpmt_enabled:1; + u16 tbt_enabled:1; + u16 psr_enabled:1; + u16 ips_enabled:1; + u16 reserved3:4; + u16 pc_feature_valid:1; } __packed; #define EDP_18BPP 0 @@ -710,45 +743,195 @@ int intel_parse_bios(struct drm_device *dev); #define DVO_PORT_DPC 8 #define DVO_PORT_DPD 9 #define DVO_PORT_DPA 10 +#define DVO_PORT_MIPIA 21 +#define DVO_PORT_MIPIB 22 +#define DVO_PORT_MIPIC 23 +#define DVO_PORT_MIPID 24 + +/* Block 52 contains MIPI Panel info + * 6 such enteries will there. Index into correct + * entery is based on the panel_index in #40 LFP + */ +#define MAX_MIPI_CONFIGURATIONS 6 + +#define MIPI_DSI_UNDEFINED_PANEL_ID 0 +#define MIPI_DSI_GENERIC_PANEL_ID 1 -/* MIPI DSI panel info */ -struct bdb_mipi { +struct mipi_config { u16 panel_id; - u16 bridge_revision; - /* General params */ - u32 dithering:1; - u32 bpp_pixel_format:1; + /* General Params */ + u32 enable_dithering:1; u32 rsvd1:1; - u32 dphy_valid:1; - u32 resvd2:28; - - u16 port_info; - u16 rsvd3:2; - u16 num_lanes:2; - u16 rsvd4:12; - - /* DSI config */ - u16 virt_ch_num:2; - u16 vtm:2; - u16 rsvd5:12; - - u32 dsi_clock; + u32 is_bridge:1; + + u32 panel_arch_type:2; + u32 is_cmd_mode:1; + +#define NON_BURST_SYNC_PULSE 0x1 +#define NON_BURST_SYNC_EVENTS 0x2 +#define BURST_MODE 0x3 + u32 video_transfer_mode:2; + + u32 cabc_supported:1; + u32 pwm_blc:1; + + /* Bit 13:10 */ +#define PIXEL_FORMAT_RGB565 0x1 +#define PIXEL_FORMAT_RGB666 0x2 +#define PIXEL_FORMAT_RGB666_LOOSELY_PACKED 0x3 +#define PIXEL_FORMAT_RGB888 0x4 + u32 videomode_color_format:4; + + /* Bit 15:14 */ +#define ENABLE_ROTATION_0 0x0 +#define ENABLE_ROTATION_90 0x1 +#define ENABLE_ROTATION_180 0x2 +#define ENABLE_ROTATION_270 0x3 + u32 rotation:2; + u32 bta_enabled:1; + u32 rsvd2:15; + + /* 2 byte Port Description */ +#define DUAL_LINK_NOT_SUPPORTED 0 +#define DUAL_LINK_FRONT_BACK 1 +#define DUAL_LINK_PIXEL_ALT 2 + u16 dual_link:2; + u16 lane_cnt:2; + u16 rsvd3:12; + + u16 rsvd4; + + u8 rsvd5[5]; + u32 dsi_ddr_clk; u32 bridge_ref_clk; - u16 rsvd_pwr; - /* Dphy Params */ - u32 prepare_cnt:5; - u32 rsvd6:3; +#define BYTE_CLK_SEL_20MHZ 0 +#define BYTE_CLK_SEL_10MHZ 1 +#define BYTE_CLK_SEL_5MHZ 2 + u8 byte_clk_sel:2; + + u8 rsvd6:6; + + /* DPHY Flags */ + u16 dphy_param_valid:1; + u16 eot_pkt_disabled:1; + u16 enable_clk_stop:1; + u16 rsvd7:13; + + u32 hs_tx_timeout; + u32 lp_rx_timeout; + u32 turn_around_timeout; + u32 device_reset_timer; + u32 master_init_timer; + u32 dbi_bw_timer; + u32 lp_byte_clk_val; + + /* 4 byte Dphy Params */ + u32 prepare_cnt:6; + u32 rsvd8:2; u32 clk_zero_cnt:8; u32 trail_cnt:5; - u32 rsvd7:3; + u32 rsvd9:3; u32 exit_zero_cnt:6; - u32 rsvd8:2; + u32 rsvd10:2; - u32 hl_switch_cnt; - u32 lp_byte_clk; u32 clk_lane_switch_cnt; + u32 hl_switch_cnt; + + u32 rsvd11[6]; + + /* timings based on dphy spec */ + u8 tclk_miss; + u8 tclk_post; + u8 rsvd12; + u8 tclk_pre; + u8 tclk_prepare; + u8 tclk_settle; + u8 tclk_term_enable; + u8 tclk_trail; + u16 tclk_prepare_clkzero; + u8 rsvd13; + u8 td_term_enable; + u8 teot; + u8 ths_exit; + u8 ths_prepare; + u16 ths_prepare_hszero; + u8 rsvd14; + u8 ths_settle; + u8 ths_skip; + u8 ths_trail; + u8 tinit; + u8 tlpx; + u8 rsvd15[3]; + + /* GPIOs */ + u8 panel_enable; + u8 bl_enable; + u8 pwm_enable; + u8 reset_r_n; + u8 pwr_down_r; + u8 stdby_r_n; + } __packed; +/* Block 52 contains MIPI configuration block + * 6 * bdb_mipi_config, followed by 6 pps data + * block below + * + * all delays has a unit of 100us + */ +struct mipi_pps_data { + u16 panel_on_delay; + u16 bl_enable_delay; + u16 bl_disable_delay; + u16 panel_off_delay; + u16 panel_power_cycle_delay; +}; + +struct bdb_mipi_config { + struct mipi_config config[MAX_MIPI_CONFIGURATIONS]; + struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS]; +}; + +/* Block 53 contains MIPI sequences as needed by the panel + * for enabling it. This block can be variable in size and + * can be maximum of 6 blocks + */ +struct bdb_mipi_sequence { + u8 version; + u8 data[0]; +}; + +/* MIPI Sequnece Block definitions */ +enum mipi_seq { + MIPI_SEQ_UNDEFINED = 0, + MIPI_SEQ_ASSERT_RESET, + MIPI_SEQ_INIT_OTP, + MIPI_SEQ_DISPLAY_ON, + MIPI_SEQ_DISPLAY_OFF, + MIPI_SEQ_DEASSERT_RESET, + MIPI_SEQ_MAX +}; + +enum mipi_seq_element { + MIPI_SEQ_ELEM_UNDEFINED = 0, + MIPI_SEQ_ELEM_SEND_PKT, + MIPI_SEQ_ELEM_DELAY, + MIPI_SEQ_ELEM_GPIO, + MIPI_SEQ_ELEM_STATUS, + MIPI_SEQ_ELEM_MAX +}; + +enum mipi_gpio_pin_index { + MIPI_GPIO_UNDEFINED = 0, + MIPI_GPIO_PANEL_ENABLE, + MIPI_GPIO_BL_ENABLE, + MIPI_GPIO_PWM_ENABLE, + MIPI_GPIO_RESET_N, + MIPI_GPIO_PWR_DOWN_R, + MIPI_GPIO_STDBY_RST_N, + MIPI_GPIO_MAX +}; + #endif /* _I830_BIOS_H_ */ diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index e2e39e65f10..5a045d3bd77 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -68,8 +68,13 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder, struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crt *crt = intel_encoder_to_crt(encoder); + enum intel_display_power_domain power_domain; u32 tmp; + power_domain = intel_display_port_power_domain(encoder); + if (!intel_display_power_enabled(dev_priv, power_domain)) + return false; + tmp = I915_READ(crt->adpa_reg); if (!(tmp & ADPA_DAC_ENABLE)) @@ -139,28 +144,49 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode) struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crt *crt = intel_encoder_to_crt(encoder); - u32 temp; + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + struct drm_display_mode *adjusted_mode = &crtc->config.adjusted_mode; + u32 adpa; - temp = I915_READ(crt->adpa_reg); - temp &= ~(ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE); - temp &= ~ADPA_DAC_ENABLE; + if (INTEL_INFO(dev)->gen >= 5) + adpa = ADPA_HOTPLUG_BITS; + else + adpa = 0; + + if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) + adpa |= ADPA_HSYNC_ACTIVE_HIGH; + if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) + adpa |= ADPA_VSYNC_ACTIVE_HIGH; + + /* For CPT allow 3 pipe config, for others just use A or B */ + if (HAS_PCH_LPT(dev)) + ; /* Those bits don't exist here */ + else if (HAS_PCH_CPT(dev)) + adpa |= PORT_TRANS_SEL_CPT(crtc->pipe); + else if (crtc->pipe == 0) + adpa |= ADPA_PIPE_A_SELECT; + else + adpa |= ADPA_PIPE_B_SELECT; + + if (!HAS_PCH_SPLIT(dev)) + I915_WRITE(BCLRPAT(crtc->pipe), 0); switch (mode) { case DRM_MODE_DPMS_ON: - temp |= ADPA_DAC_ENABLE; + adpa |= ADPA_DAC_ENABLE; break; case DRM_MODE_DPMS_STANDBY: - temp |= ADPA_DAC_ENABLE | ADPA_HSYNC_CNTL_DISABLE; + adpa |= ADPA_DAC_ENABLE | ADPA_HSYNC_CNTL_DISABLE; break; case DRM_MODE_DPMS_SUSPEND: - temp |= ADPA_DAC_ENABLE | ADPA_VSYNC_CNTL_DISABLE; + adpa |= ADPA_DAC_ENABLE | ADPA_VSYNC_CNTL_DISABLE; break; case DRM_MODE_DPMS_OFF: - temp |= ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE; + adpa |= ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE; break; } - I915_WRITE(crt->adpa_reg, temp); + I915_WRITE(crt->adpa_reg, adpa); } static void intel_disable_crt(struct intel_encoder *encoder) @@ -262,43 +288,11 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder, if (HAS_PCH_LPT(dev)) pipe_config->pipe_bpp = 24; - return true; -} - -static void intel_crt_mode_set(struct intel_encoder *encoder) -{ - - struct drm_device *dev = encoder->base.dev; - struct intel_crt *crt = intel_encoder_to_crt(encoder); - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_display_mode *adjusted_mode = &crtc->config.adjusted_mode; - u32 adpa; - - if (INTEL_INFO(dev)->gen >= 5) - adpa = ADPA_HOTPLUG_BITS; - else - adpa = 0; - - if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) - adpa |= ADPA_HSYNC_ACTIVE_HIGH; - if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) - adpa |= ADPA_VSYNC_ACTIVE_HIGH; - - /* For CPT allow 3 pipe config, for others just use A or B */ - if (HAS_PCH_LPT(dev)) - ; /* Those bits don't exist here */ - else if (HAS_PCH_CPT(dev)) - adpa |= PORT_TRANS_SEL_CPT(crtc->pipe); - else if (crtc->pipe == 0) - adpa |= ADPA_PIPE_A_SELECT; - else - adpa |= ADPA_PIPE_B_SELECT; - - if (!HAS_PCH_SPLIT(dev)) - I915_WRITE(BCLRPAT(crtc->pipe), 0); + /* FDI must always be 2.7 GHz */ + if (HAS_DDI(dev)) + pipe_config->port_clock = 135000 * 2; - I915_WRITE(crt->adpa_reg, adpa); + return true; } static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector) @@ -630,14 +624,23 @@ static enum drm_connector_status intel_crt_detect(struct drm_connector *connector, bool force) { struct drm_device *dev = connector->dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crt *crt = intel_attached_crt(connector); + struct intel_encoder *intel_encoder = &crt->base; + enum intel_display_power_domain power_domain; enum drm_connector_status status; struct intel_load_detect_pipe tmp; + struct drm_modeset_acquire_ctx ctx; + + intel_runtime_pm_get(dev_priv); DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force=%d\n", - connector->base.id, drm_get_connector_name(connector), + connector->base.id, connector->name, force); + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + if (I915_HAS_HOTPLUG(dev)) { /* We can not rely on the HPD pin always being correctly wired * up, for example many KVM do not pass it through, and so @@ -645,34 +648,45 @@ intel_crt_detect(struct drm_connector *connector, bool force) */ if (intel_crt_detect_hotplug(connector)) { DRM_DEBUG_KMS("CRT detected via hotplug\n"); - return connector_status_connected; + status = connector_status_connected; + goto out; } else DRM_DEBUG_KMS("CRT not detected via hotplug\n"); } - if (intel_crt_detect_ddc(connector)) - return connector_status_connected; + if (intel_crt_detect_ddc(connector)) { + status = connector_status_connected; + goto out; + } /* Load detection is broken on HPD capable machines. Whoever wants a * broken monitor (without edid) to work behind a broken kvm (that fails * to have the right resistors for HP detection) needs to fix this up. * For now just bail out. */ - if (I915_HAS_HOTPLUG(dev)) - return connector_status_disconnected; + if (I915_HAS_HOTPLUG(dev)) { + status = connector_status_disconnected; + goto out; + } - if (!force) - return connector->status; + if (!force) { + status = connector->status; + goto out; + } /* for pre-945g platforms use load detect */ - if (intel_get_load_detect_pipe(connector, NULL, &tmp)) { + if (intel_get_load_detect_pipe(connector, NULL, &tmp, &ctx)) { if (intel_crt_detect_ddc(connector)) status = connector_status_connected; else status = intel_crt_load_detect(crt); - intel_release_load_detect_pipe(connector, &tmp); + intel_release_load_detect_pipe(connector, &tmp, &ctx); } else status = connector_status_unknown; +out: + intel_display_power_put(dev_priv, power_domain); + intel_runtime_pm_put(dev_priv); + return status; } @@ -686,17 +700,28 @@ static int intel_crt_get_modes(struct drm_connector *connector) { struct drm_device *dev = connector->dev; struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crt *crt = intel_attached_crt(connector); + struct intel_encoder *intel_encoder = &crt->base; + enum intel_display_power_domain power_domain; int ret; struct i2c_adapter *i2c; + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin); ret = intel_crt_ddc_get_modes(connector, i2c); if (ret || !IS_G4X(dev)) - return ret; + goto out; /* Try to probe digital port for output in DVI-I -> VGA mode. */ i2c = intel_gmbus_get_adapter(dev_priv, GMBUS_PORT_DPB); - return intel_crt_ddc_get_modes(connector, i2c); + ret = intel_crt_ddc_get_modes(connector, i2c); + +out: + intel_display_power_put(dev_priv, power_domain); + + return ret; } static int intel_crt_set_property(struct drm_connector *connector, @@ -765,6 +790,14 @@ static const struct dmi_system_id intel_no_crt[] = { DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"), }, }, + { + .callback = intel_no_crt_dmi_callback, + .ident = "DELL XPS 8700", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "XPS 8700"), + }, + }, { } }; @@ -800,7 +833,7 @@ void intel_crt_init(struct drm_device *dev) intel_connector_attach_encoder(intel_connector, &crt->base); crt->base.type = INTEL_OUTPUT_ANALOG; - crt->base.cloneable = true; + crt->base.cloneable = (1 << INTEL_OUTPUT_DVO) | (1 << INTEL_OUTPUT_HDMI); if (IS_I830(dev)) crt->base.crtc_mask = (1 << 0); else @@ -820,7 +853,6 @@ void intel_crt_init(struct drm_device *dev) crt->adpa_reg = ADPA; crt->base.compute_config = intel_crt_compute_config; - crt->base.mode_set = intel_crt_mode_set; crt->base.disable = intel_disable_crt; crt->base.enable = intel_enable_crt; if (I915_HAS_HOTPLUG(dev)) @@ -833,6 +865,7 @@ void intel_crt_init(struct drm_device *dev) crt->base.get_hw_state = intel_crt_get_hw_state; } intel_connector->get_hw_state = intel_connector_get_hw_state; + intel_connector->unregister = intel_connector_unregister; drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs); @@ -857,4 +890,6 @@ void intel_crt_init(struct drm_device *dev) dev_priv->fdi_rx_config = I915_READ(_FDI_RXA_CTL) & fdi_config; } + + intel_crt_reset(connector); } diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index e06b9e017d6..b17b9c7c769 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -364,55 +364,6 @@ void hsw_fdi_link_train(struct drm_crtc *crtc) DRM_ERROR("FDI link training failed!\n"); } -static void intel_ddi_mode_set(struct intel_encoder *encoder) -{ - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - int port = intel_ddi_get_encoder_port(encoder); - int pipe = crtc->pipe; - int type = encoder->type; - struct drm_display_mode *adjusted_mode = &crtc->config.adjusted_mode; - - DRM_DEBUG_KMS("Preparing DDI mode on port %c, pipe %c\n", - port_name(port), pipe_name(pipe)); - - crtc->eld_vld = false; - if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { - struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - struct intel_digital_port *intel_dig_port = - enc_to_dig_port(&encoder->base); - - intel_dp->DP = intel_dig_port->saved_port_bits | - DDI_BUF_CTL_ENABLE | DDI_BUF_EMP_400MV_0DB_HSW; - intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count); - - if (intel_dp->has_audio) { - DRM_DEBUG_DRIVER("DP audio on pipe %c on DDI\n", - pipe_name(crtc->pipe)); - - /* write eld */ - DRM_DEBUG_DRIVER("DP audio: write eld information\n"); - intel_write_eld(&encoder->base, adjusted_mode); - } - } else if (type == INTEL_OUTPUT_HDMI) { - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); - - if (intel_hdmi->has_audio) { - /* Proper support for digital audio needs a new logic - * and a new set of registers, so we leave it for future - * patch bombing. - */ - DRM_DEBUG_DRIVER("HDMI audio on pipe %c on DDI\n", - pipe_name(crtc->pipe)); - - /* write eld */ - DRM_DEBUG_DRIVER("HDMI audio: write eld information\n"); - intel_write_eld(&encoder->base, adjusted_mode); - } - - intel_hdmi->set_infoframes(&encoder->base, adjusted_mode); - } -} - static struct intel_encoder * intel_ddi_get_crtc_encoder(struct drm_crtc *crtc) { @@ -633,6 +584,97 @@ static void wrpll_update_rnp(uint64_t freq2k, unsigned budget, /* Otherwise a < c && b >= d, do nothing */ } +static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, + int reg) +{ + int refclk = LC_FREQ; + int n, p, r; + u32 wrpll; + + wrpll = I915_READ(reg); + switch (wrpll & SPLL_PLL_REF_MASK) { + case SPLL_PLL_SSC: + case SPLL_PLL_NON_SSC: + /* + * We could calculate spread here, but our checking + * code only cares about 5% accuracy, and spread is a max of + * 0.5% downspread. + */ + refclk = 135; + break; + case SPLL_PLL_LCPLL: + refclk = LC_FREQ; + break; + default: + WARN(1, "bad wrpll refclk\n"); + return 0; + } + + r = wrpll & WRPLL_DIVIDER_REF_MASK; + p = (wrpll & WRPLL_DIVIDER_POST_MASK) >> WRPLL_DIVIDER_POST_SHIFT; + n = (wrpll & WRPLL_DIVIDER_FB_MASK) >> WRPLL_DIVIDER_FB_SHIFT; + + /* Convert to KHz, p & r have a fixed point portion */ + return (refclk * n * 100) / (p * r); +} + +static void intel_ddi_clock_get(struct intel_encoder *encoder, + struct intel_crtc_config *pipe_config) +{ + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + enum port port = intel_ddi_get_encoder_port(encoder); + int link_clock = 0; + u32 val, pll; + + val = I915_READ(PORT_CLK_SEL(port)); + switch (val & PORT_CLK_SEL_MASK) { + case PORT_CLK_SEL_LCPLL_810: + link_clock = 81000; + break; + case PORT_CLK_SEL_LCPLL_1350: + link_clock = 135000; + break; + case PORT_CLK_SEL_LCPLL_2700: + link_clock = 270000; + break; + case PORT_CLK_SEL_WRPLL1: + link_clock = intel_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL1); + break; + case PORT_CLK_SEL_WRPLL2: + link_clock = intel_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL2); + break; + case PORT_CLK_SEL_SPLL: + pll = I915_READ(SPLL_CTL) & SPLL_PLL_FREQ_MASK; + if (pll == SPLL_PLL_FREQ_810MHz) + link_clock = 81000; + else if (pll == SPLL_PLL_FREQ_1350MHz) + link_clock = 135000; + else if (pll == SPLL_PLL_FREQ_2700MHz) + link_clock = 270000; + else { + WARN(1, "bad spll freq\n"); + return; + } + break; + default: + WARN(1, "bad port clock sel\n"); + return; + } + + pipe_config->port_clock = link_clock * 2; + + if (pipe_config->has_pch_encoder) + pipe_config->adjusted_mode.crtc_clock = + intel_dotclock_calculate(pipe_config->port_clock, + &pipe_config->fdi_m_n); + else if (pipe_config->has_dp_encoder) + pipe_config->adjusted_mode.crtc_clock = + intel_dotclock_calculate(pipe_config->port_clock, + &pipe_config->dp_m_n); + else + pipe_config->adjusted_mode.crtc_clock = pipe_config->port_clock; +} + static void intel_ddi_calculate_wrpll(int clock /* in Hz */, unsigned *r2_out, unsigned *n2_out, unsigned *p_out) @@ -971,9 +1013,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc) } if (type == INTEL_OUTPUT_HDMI) { - struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); - - if (intel_hdmi->has_hdmi_sink) + if (intel_crtc->config.has_hdmi_sink) temp |= TRANS_DDI_MODE_SELECT_HDMI; else temp |= TRANS_DDI_MODE_SELECT_DVI; @@ -1017,8 +1057,13 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) enum port port = intel_ddi_get_encoder_port(intel_encoder); enum pipe pipe = 0; enum transcoder cpu_transcoder; + enum intel_display_power_domain power_domain; uint32_t tmp; + power_domain = intel_display_port_power_domain(intel_encoder); + if (!intel_display_power_enabled(dev_priv, power_domain)) + return false; + if (!intel_encoder->get_hw_state(intel_encoder, &pipe)) return false; @@ -1054,9 +1099,14 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum port port = intel_ddi_get_encoder_port(encoder); + enum intel_display_power_domain power_domain; u32 tmp; int i; + power_domain = intel_display_port_power_domain(encoder); + if (!intel_display_power_enabled(dev_priv, power_domain)) + return false; + tmp = I915_READ(DDI_BUF_CTL(port)); if (!(tmp & DDI_BUF_CTL_ENABLE)) @@ -1192,28 +1242,48 @@ void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc) static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder) { struct drm_encoder *encoder = &intel_encoder->base; - struct drm_crtc *crtc = encoder->crtc; struct drm_i915_private *dev_priv = encoder->dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); enum port port = intel_ddi_get_encoder_port(intel_encoder); int type = intel_encoder->type; + if (crtc->config.has_audio) { + DRM_DEBUG_DRIVER("Audio on pipe %c on DDI\n", + pipe_name(crtc->pipe)); + + /* write eld */ + DRM_DEBUG_DRIVER("DDI audio: write eld information\n"); + intel_write_eld(encoder, &crtc->config.adjusted_mode); + } + if (type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - ironlake_edp_panel_on(intel_dp); + intel_edp_panel_on(intel_dp); } - WARN_ON(intel_crtc->ddi_pll_sel == PORT_CLK_SEL_NONE); - I915_WRITE(PORT_CLK_SEL(port), intel_crtc->ddi_pll_sel); + WARN_ON(crtc->ddi_pll_sel == PORT_CLK_SEL_NONE); + I915_WRITE(PORT_CLK_SEL(port), crtc->ddi_pll_sel); if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + struct intel_digital_port *intel_dig_port = + enc_to_dig_port(encoder); + + intel_dp->DP = intel_dig_port->saved_port_bits | + DDI_BUF_CTL_ENABLE | DDI_BUF_EMP_400MV_0DB_HSW; + intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); intel_dp_start_link_train(intel_dp); intel_dp_complete_link_train(intel_dp); if (port != PORT_A) intel_dp_stop_link_train(intel_dp); + } else if (type == INTEL_OUTPUT_HDMI) { + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); + + intel_hdmi->set_infoframes(encoder, + crtc->config.has_hdmi_sink, + &crtc->config.adjusted_mode); } } @@ -1244,7 +1314,8 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder) if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); - ironlake_edp_panel_off(intel_dp); + intel_edp_panel_vdd_on(intel_dp); + intel_edp_panel_off(intel_dp); } I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE); @@ -1279,11 +1350,12 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder) if (port == PORT_A) intel_dp_stop_link_train(intel_dp); - ironlake_edp_backlight_on(intel_dp); + intel_edp_backlight_on(intel_dp); intel_edp_psr_enable(intel_dp); } - if (intel_crtc->eld_vld && type != INTEL_OUTPUT_EDP) { + if (intel_crtc->config.has_audio) { + intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); tmp |= ((AUDIO_OUTPUT_ENABLE_A | AUDIO_ELD_VALID_A) << (pipe * 4)); I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp); @@ -1301,18 +1373,21 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder) struct drm_i915_private *dev_priv = dev->dev_private; uint32_t tmp; - if (intel_crtc->eld_vld && type != INTEL_OUTPUT_EDP) { + /* We can't touch HSW_AUD_PIN_ELD_CP_VLD uncionditionally because this + * register is part of the power well on Haswell. */ + if (intel_crtc->config.has_audio) { tmp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); tmp &= ~((AUDIO_OUTPUT_ENABLE_A | AUDIO_ELD_VALID_A) << (pipe * 4)); I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp); + intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); } if (type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); intel_edp_psr_disable(intel_dp); - ironlake_edp_backlight_off(intel_dp); + intel_edp_backlight_off(intel_dp); } } @@ -1324,7 +1399,7 @@ int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv) if (lcpll & LCPLL_CD_SOURCE_FCLK) { return 800000; - } else if (I915_READ(HSW_FUSE_STRAP) & HSW_CDCLK_LIMIT) { + } else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) { return 450000; } else if (freq == LCPLL_CLK_FREQ_450) { return 450000; @@ -1478,6 +1553,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder, switch (temp & TRANS_DDI_MODE_SELECT_MASK) { case TRANS_DDI_MODE_SELECT_HDMI: + pipe_config->has_hdmi_sink = true; case TRANS_DDI_MODE_SELECT_DVI: case TRANS_DDI_MODE_SELECT_FDI: break; @@ -1490,6 +1566,12 @@ void intel_ddi_get_config(struct intel_encoder *encoder, break; } + if (intel_display_power_enabled(dev_priv, POWER_DOMAIN_AUDIO)) { + temp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD); + if (temp & (AUDIO_OUTPUT_ENABLE_A << (intel_crtc->pipe * 4))) + pipe_config->has_audio = true; + } + if (encoder->type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp_bpp && pipe_config->pipe_bpp > dev_priv->vbt.edp_bpp) { /* @@ -1509,6 +1591,8 @@ void intel_ddi_get_config(struct intel_encoder *encoder, pipe_config->pipe_bpp, dev_priv->vbt.edp_bpp); dev_priv->vbt.edp_bpp = pipe_config->pipe_bpp; } + + intel_ddi_clock_get(encoder, pipe_config); } static void intel_ddi_destroy(struct drm_encoder *encoder) @@ -1604,7 +1688,6 @@ void intel_ddi_init(struct drm_device *dev, enum port port) DRM_MODE_ENCODER_TMDS); intel_encoder->compute_config = intel_ddi_compute_config; - intel_encoder->mode_set = intel_ddi_mode_set; intel_encoder->enable = intel_enable_ddi; intel_encoder->pre_enable = intel_ddi_pre_enable; intel_encoder->disable = intel_disable_ddi; @@ -1619,7 +1702,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port) intel_encoder->type = INTEL_OUTPUT_UNKNOWN; intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); - intel_encoder->cloneable = false; + intel_encoder->cloneable = 0; intel_encoder->hot_plug = intel_ddi_hot_plug; if (init_dp) diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 9fa24347963..f0be855ddf4 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -41,6 +41,9 @@ #include <drm/drm_crtc_helper.h> #include <linux/dma_remapping.h> +#define DIV_ROUND_CLOSEST_ULL(ll, d) \ + ({ unsigned long long _tmp = (ll)+(d)/2; do_div(_tmp, d); _tmp; }) + static void intel_increase_pllclock(struct drm_crtc *crtc); static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on); @@ -51,7 +54,19 @@ static void ironlake_pch_clock_get(struct intel_crtc *crtc, static int intel_set_mode(struct drm_crtc *crtc, struct drm_display_mode *mode, int x, int y, struct drm_framebuffer *old_fb); - +static int intel_framebuffer_init(struct drm_device *dev, + struct intel_framebuffer *ifb, + struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_i915_gem_object *obj); +static void intel_dp_set_m_n(struct intel_crtc *crtc); +static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc); +static void intel_set_pipe_timings(struct intel_crtc *intel_crtc); +static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, + struct intel_link_m_n *m_n); +static void ironlake_set_pipeconf(struct drm_crtc *crtc); +static void haswell_set_pipeconf(struct drm_crtc *crtc); +static void intel_set_pipe_csc(struct drm_crtc *crtc); +static void vlv_prepare_pll(struct intel_crtc *crtc); typedef struct { int min, max; @@ -325,6 +340,22 @@ static const intel_limit_t intel_limits_vlv = { .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */ }; +static const intel_limit_t intel_limits_chv = { + /* + * These are the data rate limits (measured in fast clocks) + * since those are the strictest limits we have. The fast + * clock and actual rate limits are more relaxed, so checking + * them would make no difference. + */ + .dot = { .min = 25000 * 5, .max = 540000 * 5}, + .vco = { .min = 4860000, .max = 6700000 }, + .n = { .min = 1, .max = 1 }, + .m1 = { .min = 2, .max = 2 }, + .m2 = { .min = 24 << 22, .max = 175 << 22 }, + .p1 = { .min = 2, .max = 4 }, + .p2 = { .p2_slow = 1, .p2_fast = 14 }, +}; + static void vlv_clock(int refclk, intel_clock_t *clock) { clock->m = clock->m1 * clock->m2; @@ -409,6 +440,8 @@ static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk) limit = &intel_limits_pineview_lvds; else limit = &intel_limits_pineview_sdvo; + } else if (IS_CHERRYVIEW(dev)) { + limit = &intel_limits_chv; } else if (IS_VALLEYVIEW(dev)) { limit = &intel_limits_vlv; } else if (!IS_GEN2(dev)) { @@ -453,6 +486,17 @@ static void i9xx_clock(int refclk, intel_clock_t *clock) clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); } +static void chv_clock(int refclk, intel_clock_t *clock) +{ + clock->m = clock->m1 * clock->m2; + clock->p = clock->p1 * clock->p2; + if (WARN_ON(clock->n == 0 || clock->p == 0)) + return; + clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m, + clock->n << 22); + clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p); +} + #define INTELPllInvalid(s) do { /* DRM_DEBUG(s); */ return false; } while (0) /** * Returns whether the given set of divisors are valid for a given refclk with @@ -728,6 +772,58 @@ vlv_find_best_dpll(const intel_limit_t *limit, struct drm_crtc *crtc, return found; } +static bool +chv_find_best_dpll(const intel_limit_t *limit, struct drm_crtc *crtc, + int target, int refclk, intel_clock_t *match_clock, + intel_clock_t *best_clock) +{ + struct drm_device *dev = crtc->dev; + intel_clock_t clock; + uint64_t m2; + int found = false; + + memset(best_clock, 0, sizeof(*best_clock)); + + /* + * Based on hardware doc, the n always set to 1, and m1 always + * set to 2. If requires to support 200Mhz refclk, we need to + * revisit this because n may not 1 anymore. + */ + clock.n = 1, clock.m1 = 2; + target *= 5; /* fast clock */ + + for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) { + for (clock.p2 = limit->p2.p2_fast; + clock.p2 >= limit->p2.p2_slow; + clock.p2 -= clock.p2 > 10 ? 2 : 1) { + + clock.p = clock.p1 * clock.p2; + + m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p * + clock.n) << 22, refclk * clock.m1); + + if (m2 > INT_MAX/clock.m1) + continue; + + clock.m2 = m2; + + chv_clock(refclk, &clock); + + if (!intel_PLL_is_valid(dev, limit, &clock)) + continue; + + /* based on hardware requirement, prefer bigger p + */ + if (clock.p > best_clock->p) { + *best_clock = clock; + found = true; + } + } + } + + return found; +} + bool intel_crtc_active(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -738,10 +834,10 @@ bool intel_crtc_active(struct drm_crtc *crtc) * We can ditch the adjusted_mode.crtc_clock check as soon * as Haswell has gained clock readout/fastboot support. * - * We can ditch the crtc->fb check as soon as we can + * We can ditch the crtc->primary->fb check as soon as we can * properly reconstruct framebuffers. */ - return intel_crtc->active && crtc->fb && + return intel_crtc->active && crtc->primary->fb && intel_crtc->config.adjusted_mode.crtc_clock; } @@ -762,7 +858,7 @@ static void g4x_wait_for_vblank(struct drm_device *dev, int pipe) frame = I915_READ(frame_reg); if (wait_for(I915_READ_NOTRACE(frame_reg) != frame, 50)) - DRM_DEBUG_KMS("vblank wait timed out\n"); + WARN(1, "vblank wait timed out\n"); } /** @@ -875,7 +971,7 @@ bool ibx_digital_port_connected(struct drm_i915_private *dev_priv, u32 bit; if (HAS_PCH_IBX(dev_priv->dev)) { - switch(port->port) { + switch (port->port) { case PORT_B: bit = SDE_PORTB_HOTPLUG; break; @@ -889,7 +985,7 @@ bool ibx_digital_port_connected(struct drm_i915_private *dev_priv, return true; } } else { - switch(port->port) { + switch (port->port) { case PORT_B: bit = SDE_PORTB_HOTPLUG_CPT; break; @@ -1030,7 +1126,7 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, u32 val; /* ILK FDI PLL is always enabled */ - if (dev_priv->info->gen == 5) + if (INTEL_INFO(dev_priv->dev)->gen == 5) return; /* On Haswell, DDI ports are responsible for the FDI PLL setup */ @@ -1092,9 +1188,7 @@ static void assert_cursor(struct drm_i915_private *dev_priv, struct drm_device *dev = dev_priv->dev; bool cur_state; - if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) - cur_state = I915_READ(CURCNTR_IVB(pipe)) & CURSOR_MODE; - else if (IS_845G(dev) || IS_I865G(dev)) + if (IS_845G(dev) || IS_I865G(dev)) cur_state = I915_READ(_CURACNTR) & CURSOR_ENABLE; else cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE; @@ -1119,7 +1213,7 @@ void assert_pipe(struct drm_i915_private *dev_priv, if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) state = true; - if (!intel_display_power_enabled(dev_priv->dev, + if (!intel_display_power_enabled(dev_priv, POWER_DOMAIN_TRANSCODER(cpu_transcoder))) { cur_state = false; } else { @@ -1163,7 +1257,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv, if (INTEL_INFO(dev)->gen >= 4) { reg = DSPCNTR(pipe); val = I915_READ(reg); - WARN((val & DISPLAY_PLANE_ENABLE), + WARN(val & DISPLAY_PLANE_ENABLE, "plane %c assertion failure, should be disabled but not\n", plane_name(pipe)); return; @@ -1185,27 +1279,27 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { struct drm_device *dev = dev_priv->dev; - int reg, i; + int reg, sprite; u32 val; if (IS_VALLEYVIEW(dev)) { - for (i = 0; i < dev_priv->num_plane; i++) { - reg = SPCNTR(pipe, i); + for_each_sprite(pipe, sprite) { + reg = SPCNTR(pipe, sprite); val = I915_READ(reg); - WARN((val & SP_ENABLE), + WARN(val & SP_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", - sprite_name(pipe, i), pipe_name(pipe)); + sprite_name(pipe, sprite), pipe_name(pipe)); } } else if (INTEL_INFO(dev)->gen >= 7) { reg = SPRCTL(pipe); val = I915_READ(reg); - WARN((val & SPRITE_ENABLE), + WARN(val & SPRITE_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", plane_name(pipe), pipe_name(pipe)); } else if (INTEL_INFO(dev)->gen >= 5) { reg = DVSCNTR(pipe); val = I915_READ(reg); - WARN((val & DVS_ENABLE), + WARN(val & DVS_ENABLE, "sprite %c assertion failure, should be off on pipe %c but is still active\n", plane_name(pipe), pipe_name(pipe)); } @@ -1250,6 +1344,9 @@ static bool dp_pipe_enabled(struct drm_i915_private *dev_priv, u32 trans_dp_ctl = I915_READ(trans_dp_ctl_reg); if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel) return false; + } else if (IS_CHERRYVIEW(dev_priv->dev)) { + if ((val & DP_PIPE_MASK_CHV) != DP_PIPE_SELECT_CHV(pipe)) + return false; } else { if ((val & DP_PIPE_MASK) != (pipe << 30)) return false; @@ -1266,6 +1363,9 @@ static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv, if (HAS_PCH_CPT(dev_priv->dev)) { if ((val & SDVO_PIPE_SEL_MASK_CPT) != SDVO_PIPE_SEL_CPT(pipe)) return false; + } else if (IS_CHERRYVIEW(dev_priv->dev)) { + if ((val & SDVO_PIPE_SEL_MASK_CHV) != SDVO_PIPE_SEL_CHV(pipe)) + return false; } else { if ((val & SDVO_PIPE_SEL_MASK) != SDVO_PIPE_SEL(pipe)) return false; @@ -1364,7 +1464,17 @@ static void intel_init_dpio(struct drm_device *dev) if (!IS_VALLEYVIEW(dev)) return; - DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; + /* + * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), + * CHV x1 PHY (DP/HDMI D) + * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) + */ + if (IS_CHERRYVIEW(dev)) { + DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; + DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; + } else { + DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; + } } static void intel_reset_dpio(struct drm_device *dev) @@ -1374,25 +1484,48 @@ static void intel_reset_dpio(struct drm_device *dev) if (!IS_VALLEYVIEW(dev)) return; - /* - * Enable the CRI clock source so we can get at the display and the - * reference clock for VGA hotplug / manual detection. - */ - I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | - DPLL_REFA_CLK_ENABLE_VLV | - DPLL_INTEGRATED_CRI_CLK_VLV); + if (IS_CHERRYVIEW(dev)) { + enum dpio_phy phy; + u32 val; - /* - * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx - - * 6. De-assert cmn_reset/side_reset. Same as VLV X0. - * a. GUnit 0x2110 bit[0] set to 1 (def 0) - * b. The other bits such as sfr settings / modesel may all be set - * to 0. - * - * This should only be done on init and resume from S3 with both - * PLLs disabled, or we risk losing DPIO and PLL synchronization. - */ - I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST); + for (phy = DPIO_PHY0; phy < I915_NUM_PHYS_VLV; phy++) { + /* Poll for phypwrgood signal */ + if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & + PHY_POWERGOOD(phy), 1)) + DRM_ERROR("Display PHY %d is not power up\n", phy); + + /* + * Deassert common lane reset for PHY. + * + * This should only be done on init and resume from S3 + * with both PLLs disabled, or we risk losing DPIO and + * PLL synchronization. + */ + val = I915_READ(DISPLAY_PHY_CONTROL); + I915_WRITE(DISPLAY_PHY_CONTROL, + PHY_COM_LANE_RESET_DEASSERT(phy, val)); + } + + } else { + /* + * If DPIO has already been reset, e.g. by BIOS, just skip all + * this. + */ + if (I915_READ(DPIO_CTL) & DPIO_CMNRST) + return; + + /* + * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx: + * Need to assert and de-assert PHY SB reset by gating the + * common lane power, then un-gating it. + * Simply ungating isn't enough to reset the PHY enough to get + * ports and lanes running. + */ + __vlv_set_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC, + false); + __vlv_set_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC, + true); + } } static void vlv_enable_pll(struct intel_crtc *crtc) @@ -1433,6 +1566,44 @@ static void vlv_enable_pll(struct intel_crtc *crtc) udelay(150); /* wait for warmup */ } +static void chv_enable_pll(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe = crtc->pipe; + enum dpio_channel port = vlv_pipe_to_channel(pipe); + u32 tmp; + + assert_pipe_disabled(dev_priv, crtc->pipe); + + BUG_ON(!IS_CHERRYVIEW(dev_priv->dev)); + + mutex_lock(&dev_priv->dpio_lock); + + /* Enable back the 10bit clock to display controller */ + tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)); + tmp |= DPIO_DCLKP_EN; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp); + + /* + * Need to wait > 100ns between dclkp clock enable bit and PLL enable. + */ + udelay(1); + + /* Enable PLL */ + I915_WRITE(DPLL(pipe), crtc->config.dpll_hw_state.dpll); + + /* Check PLL is locked */ + if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1)) + DRM_ERROR("PLL %d failed to lock\n", pipe); + + /* not sure when this should be written */ + I915_WRITE(DPLL_MD(pipe), crtc->config.dpll_hw_state.dpll_md); + POSTING_READ(DPLL_MD(pipe)); + + mutex_unlock(&dev_priv->dpio_lock); +} + static void i9xx_enable_pll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -1443,7 +1614,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) assert_pipe_disabled(dev_priv, crtc->pipe); /* No really, not for ILK+ */ - BUG_ON(dev_priv->info->gen >= 5); + BUG_ON(INTEL_INFO(dev)->gen >= 5); /* PLL is protected by panel, make sure we can write it */ if (IS_MOBILE(dev) && !IS_I830(dev)) @@ -1516,44 +1687,92 @@ static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) val = DPLL_INTEGRATED_CRI_CLK_VLV | DPLL_REFA_CLK_ENABLE_VLV; I915_WRITE(DPLL(pipe), val); POSTING_READ(DPLL(pipe)); + +} + +static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe) +{ + enum dpio_channel port = vlv_pipe_to_channel(pipe); + u32 val; + + /* Make sure the pipe isn't still relying on us */ + assert_pipe_disabled(dev_priv, pipe); + + /* Set PLL en = 0 */ + val = DPLL_SSC_REF_CLOCK_CHV; + if (pipe != PIPE_A) + val |= DPLL_INTEGRATED_CRI_CLK_VLV; + I915_WRITE(DPLL(pipe), val); + POSTING_READ(DPLL(pipe)); + + mutex_lock(&dev_priv->dpio_lock); + + /* Disable 10bit clock to display controller */ + val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)); + val &= ~DPIO_DCLKP_EN; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val); + + mutex_unlock(&dev_priv->dpio_lock); } void vlv_wait_port_ready(struct drm_i915_private *dev_priv, struct intel_digital_port *dport) { u32 port_mask; + int dpll_reg; switch (dport->port) { case PORT_B: port_mask = DPLL_PORTB_READY_MASK; + dpll_reg = DPLL(0); break; case PORT_C: port_mask = DPLL_PORTC_READY_MASK; + dpll_reg = DPLL(0); + break; + case PORT_D: + port_mask = DPLL_PORTD_READY_MASK; + dpll_reg = DPIO_PHY_STATUS; break; default: BUG(); } - if (wait_for((I915_READ(DPLL(0)) & port_mask) == 0, 1000)) + if (wait_for((I915_READ(dpll_reg) & port_mask) == 0, 1000)) WARN(1, "timed out waiting for port %c ready: 0x%08x\n", - port_name(dport->port), I915_READ(DPLL(0))); + port_name(dport->port), I915_READ(dpll_reg)); +} + +static void intel_prepare_shared_dpll(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc); + + WARN_ON(!pll->refcount); + if (pll->active == 0) { + DRM_DEBUG_DRIVER("setting up %s\n", pll->name); + WARN_ON(pll->on); + assert_shared_dpll_disabled(dev_priv, pll); + + pll->mode_set(dev_priv, pll); + } } /** - * ironlake_enable_shared_dpll - enable PCH PLL + * intel_enable_shared_dpll - enable PCH PLL * @dev_priv: i915 private structure * @pipe: pipe PLL to enable * * The PCH PLL needs to be enabled before the PCH transcoder, since it * drives the transcoder clock. */ -static void ironlake_enable_shared_dpll(struct intel_crtc *crtc) +static void intel_enable_shared_dpll(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc); - /* PCH PLLs only available on ILK, SNB and IVB */ - BUG_ON(dev_priv->info->gen < 5); if (WARN_ON(pll == NULL)) return; @@ -1578,11 +1797,12 @@ static void ironlake_enable_shared_dpll(struct intel_crtc *crtc) static void intel_disable_shared_dpll(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc); /* PCH only available on ILK+ */ - BUG_ON(dev_priv->info->gen < 5); + BUG_ON(INTEL_INFO(dev)->gen < 5); if (WARN_ON(pll == NULL)) return; @@ -1617,7 +1837,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, uint32_t reg, val, pipeconf_val; /* PCH only available on ILK+ */ - BUG_ON(dev_priv->info->gen < 5); + BUG_ON(INTEL_INFO(dev)->gen < 5); /* Make sure PCH DPLL is enabled */ assert_shared_dpll_enabled(dev_priv, @@ -1670,7 +1890,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv, u32 val, pipeconf_val; /* PCH only available on ILK+ */ - BUG_ON(dev_priv->info->gen < 5); + BUG_ON(INTEL_INFO(dev_priv->dev)->gen < 5); /* FDI must be feeding us bits for PCH ports */ assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder); @@ -1744,21 +1964,16 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv) /** * intel_enable_pipe - enable a pipe, asserting requirements - * @dev_priv: i915 private structure - * @pipe: pipe to enable - * @pch_port: on ILK+, is this pipe driving a PCH port or not + * @crtc: crtc responsible for the pipe * - * Enable @pipe, making sure that various hardware specific requirements + * Enable @crtc's pipe, making sure that various hardware specific requirements * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc. - * - * @pipe should be %PIPE_A or %PIPE_B. - * - * Will wait until the pipe is actually running (i.e. first vblank) before - * returning. */ -static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, - bool pch_port, bool dsi) +static void intel_enable_pipe(struct intel_crtc *crtc) { + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum pipe pipe = crtc->pipe; enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv, pipe); enum pipe pch_transcoder; @@ -1780,12 +1995,12 @@ static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, * need the check. */ if (!HAS_PCH_SPLIT(dev_priv->dev)) - if (dsi) + if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DSI)) assert_dsi_pll_enabled(dev_priv); else assert_pll_enabled(dev_priv, pipe); else { - if (pch_port) { + if (crtc->config.has_pch_encoder) { /* if driving the PCH, we need FDI enabled */ assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder); assert_fdi_tx_pll_enabled(dev_priv, @@ -1796,11 +2011,14 @@ static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, reg = PIPECONF(cpu_transcoder); val = I915_READ(reg); - if (val & PIPECONF_ENABLE) + if (val & PIPECONF_ENABLE) { + WARN_ON(!(pipe == PIPE_A && + dev_priv->quirks & QUIRK_PIPEA_FORCE)); return; + } I915_WRITE(reg, val | PIPECONF_ENABLE); - intel_wait_for_vblank(dev_priv->dev, pipe); + POSTING_READ(reg); } /** @@ -1851,23 +2069,25 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv, void intel_flush_primary_plane(struct drm_i915_private *dev_priv, enum plane plane) { - u32 reg = dev_priv->info->gen >= 4 ? DSPSURF(plane) : DSPADDR(plane); + struct drm_device *dev = dev_priv->dev; + u32 reg = INTEL_INFO(dev)->gen >= 4 ? DSPSURF(plane) : DSPADDR(plane); I915_WRITE(reg, I915_READ(reg)); POSTING_READ(reg); } /** - * intel_enable_primary_plane - enable the primary plane on a given pipe + * intel_enable_primary_hw_plane - enable the primary plane on a given pipe * @dev_priv: i915 private structure * @plane: plane to enable * @pipe: pipe being fed * * Enable @plane on @pipe, making sure that @pipe is running first. */ -static void intel_enable_primary_plane(struct drm_i915_private *dev_priv, - enum plane plane, enum pipe pipe) +static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv, + enum plane plane, enum pipe pipe) { + struct drm_device *dev = dev_priv->dev; struct intel_crtc *intel_crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); int reg; @@ -1876,48 +2096,54 @@ static void intel_enable_primary_plane(struct drm_i915_private *dev_priv, /* If the pipe isn't enabled, we can't pump pixels and may hang */ assert_pipe_enabled(dev_priv, pipe); - WARN(intel_crtc->primary_enabled, "Primary plane already enabled\n"); + if (intel_crtc->primary_enabled) + return; intel_crtc->primary_enabled = true; reg = DSPCNTR(plane); val = I915_READ(reg); - if (val & DISPLAY_PLANE_ENABLE) - return; + WARN_ON(val & DISPLAY_PLANE_ENABLE); I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE); intel_flush_primary_plane(dev_priv, plane); - intel_wait_for_vblank(dev_priv->dev, pipe); + + /* + * BDW signals flip done immediately if the plane + * is disabled, even if the plane enable is already + * armed to occur at the next vblank :( + */ + if (IS_BROADWELL(dev)) + intel_wait_for_vblank(dev, intel_crtc->pipe); } /** - * intel_disable_primary_plane - disable the primary plane + * intel_disable_primary_hw_plane - disable the primary hardware plane * @dev_priv: i915 private structure * @plane: plane to disable * @pipe: pipe consuming the data * * Disable @plane; should be an independent operation. */ -static void intel_disable_primary_plane(struct drm_i915_private *dev_priv, - enum plane plane, enum pipe pipe) +static void intel_disable_primary_hw_plane(struct drm_i915_private *dev_priv, + enum plane plane, enum pipe pipe) { struct intel_crtc *intel_crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); int reg; u32 val; - WARN(!intel_crtc->primary_enabled, "Primary plane already disabled\n"); + if (!intel_crtc->primary_enabled) + return; intel_crtc->primary_enabled = false; reg = DSPCNTR(plane); val = I915_READ(reg); - if ((val & DISPLAY_PLANE_ENABLE) == 0) - return; + WARN_ON((val & DISPLAY_PLANE_ENABLE) == 0); I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE); intel_flush_primary_plane(dev_priv, plane); - intel_wait_for_vblank(dev_priv->dev, pipe); } static bool need_vtd_wa(struct drm_device *dev) @@ -1929,10 +2155,18 @@ static bool need_vtd_wa(struct drm_device *dev) return false; } +static int intel_align_height(struct drm_device *dev, int height, bool tiled) +{ + int tile_height; + + tile_height = tiled ? (IS_GEN2(dev) ? 16 : 8) : 1; + return ALIGN(height, tile_height); +} + int intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined) + struct intel_engine_cs *pipelined) { struct drm_i915_private *dev_priv = dev->dev_private; u32 alignment; @@ -2025,8 +2259,114 @@ unsigned long intel_gen4_compute_page_offset(int *x, int *y, } } -static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb, - int x, int y) +int intel_format_to_fourcc(int format) +{ + switch (format) { + case DISPPLANE_8BPP: + return DRM_FORMAT_C8; + case DISPPLANE_BGRX555: + return DRM_FORMAT_XRGB1555; + case DISPPLANE_BGRX565: + return DRM_FORMAT_RGB565; + default: + case DISPPLANE_BGRX888: + return DRM_FORMAT_XRGB8888; + case DISPPLANE_RGBX888: + return DRM_FORMAT_XBGR8888; + case DISPPLANE_BGRX101010: + return DRM_FORMAT_XRGB2101010; + case DISPPLANE_RGBX101010: + return DRM_FORMAT_XBGR2101010; + } +} + +static bool intel_alloc_plane_obj(struct intel_crtc *crtc, + struct intel_plane_config *plane_config) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_gem_object *obj = NULL; + struct drm_mode_fb_cmd2 mode_cmd = { 0 }; + u32 base = plane_config->base; + + if (plane_config->size == 0) + return false; + + obj = i915_gem_object_create_stolen_for_preallocated(dev, base, base, + plane_config->size); + if (!obj) + return false; + + if (plane_config->tiled) { + obj->tiling_mode = I915_TILING_X; + obj->stride = crtc->base.primary->fb->pitches[0]; + } + + mode_cmd.pixel_format = crtc->base.primary->fb->pixel_format; + mode_cmd.width = crtc->base.primary->fb->width; + mode_cmd.height = crtc->base.primary->fb->height; + mode_cmd.pitches[0] = crtc->base.primary->fb->pitches[0]; + + mutex_lock(&dev->struct_mutex); + + if (intel_framebuffer_init(dev, to_intel_framebuffer(crtc->base.primary->fb), + &mode_cmd, obj)) { + DRM_DEBUG_KMS("intel fb init failed\n"); + goto out_unref_obj; + } + + mutex_unlock(&dev->struct_mutex); + + DRM_DEBUG_KMS("plane fb obj %p\n", obj); + return true; + +out_unref_obj: + drm_gem_object_unreference(&obj->base); + mutex_unlock(&dev->struct_mutex); + return false; +} + +static void intel_find_plane_obj(struct intel_crtc *intel_crtc, + struct intel_plane_config *plane_config) +{ + struct drm_device *dev = intel_crtc->base.dev; + struct drm_crtc *c; + struct intel_crtc *i; + struct intel_framebuffer *fb; + + if (!intel_crtc->base.primary->fb) + return; + + if (intel_alloc_plane_obj(intel_crtc, plane_config)) + return; + + kfree(intel_crtc->base.primary->fb); + intel_crtc->base.primary->fb = NULL; + + /* + * Failed to alloc the obj, check to see if we should share + * an fb with another CRTC instead + */ + for_each_crtc(dev, c) { + i = to_intel_crtc(c); + + if (c == &intel_crtc->base) + continue; + + if (!i->active || !c->primary->fb) + continue; + + fb = to_intel_framebuffer(c->primary->fb); + if (i915_gem_obj_ggtt_offset(fb->obj) == plane_config->base) { + drm_framebuffer_reference(c->primary->fb); + intel_crtc->base.primary->fb = c->primary->fb; + break; + } + } +} + +static void i9xx_update_primary_plane(struct drm_crtc *crtc, + struct drm_framebuffer *fb, + int x, int y) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -2038,15 +2378,6 @@ static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb, u32 dspcntr; u32 reg; - switch (plane) { - case 0: - case 1: - break; - default: - DRM_ERROR("Can't update plane %c in SAREA\n", plane_name(plane)); - return -EINVAL; - } - intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; @@ -2121,12 +2452,11 @@ static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb, } else I915_WRITE(DSPADDR(plane), i915_gem_obj_ggtt_offset(obj) + linear_offset); POSTING_READ(reg); - - return 0; } -static int ironlake_update_plane(struct drm_crtc *crtc, - struct drm_framebuffer *fb, int x, int y) +static void ironlake_update_primary_plane(struct drm_crtc *crtc, + struct drm_framebuffer *fb, + int x, int y) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -2138,16 +2468,6 @@ static int ironlake_update_plane(struct drm_crtc *crtc, u32 dspcntr; u32 reg; - switch (plane) { - case 0: - case 1: - case 2: - break; - default: - DRM_ERROR("Can't update plane %c in SAREA\n", plane_name(plane)); - return -EINVAL; - } - intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; @@ -2214,8 +2534,6 @@ static int ironlake_update_plane(struct drm_crtc *crtc, I915_WRITE(DSPLINOFF(plane), linear_offset); } POSTING_READ(reg); - - return 0; } /* Assume fb object is pinned & idle & fenced and just update base pointers */ @@ -2230,7 +2548,9 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, dev_priv->display.disable_fbc(dev); intel_increase_pllclock(crtc); - return dev_priv->display.update_plane(crtc, fb, x, y); + dev_priv->display.update_primary_plane(crtc, fb, x, y); + + return 0; } void intel_display_handle_reset(struct drm_device *dev) @@ -2252,7 +2572,7 @@ void intel_display_handle_reset(struct drm_device *dev) * pending_flip_queue really got woken up. */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + for_each_crtc(dev, crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum plane plane = intel_crtc->plane; @@ -2260,19 +2580,21 @@ void intel_display_handle_reset(struct drm_device *dev) intel_finish_page_flip_plane(dev, plane); } - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + for_each_crtc(dev, crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - mutex_lock(&crtc->mutex); + drm_modeset_lock(&crtc->mutex, NULL); /* * FIXME: Once we have proper support for primary planes (and * disabling them without disabling the entire crtc) allow again - * a NULL crtc->fb. + * a NULL crtc->primary->fb. */ - if (intel_crtc->active && crtc->fb) - dev_priv->display.update_plane(crtc, crtc->fb, - crtc->x, crtc->y); - mutex_unlock(&crtc->mutex); + if (intel_crtc->active && crtc->primary->fb) + dev_priv->display.update_primary_plane(crtc, + crtc->primary->fb, + crtc->x, + crtc->y); + drm_modeset_unlock(&crtc->mutex); } } @@ -2299,31 +2621,23 @@ intel_finish_fb(struct drm_framebuffer *old_fb) return ret; } -static void intel_crtc_update_sarea_pos(struct drm_crtc *crtc, int x, int y) +static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_master_private *master_priv; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + unsigned long flags; + bool pending; - if (!dev->primary->master) - return; + if (i915_reset_in_progress(&dev_priv->gpu_error) || + intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) + return false; - master_priv = dev->primary->master->driver_priv; - if (!master_priv->sarea_priv) - return; + spin_lock_irqsave(&dev->event_lock, flags); + pending = to_intel_crtc(crtc)->unpin_work != NULL; + spin_unlock_irqrestore(&dev->event_lock, flags); - switch (intel_crtc->pipe) { - case 0: - master_priv->sarea_priv->pipeA_x = x; - master_priv->sarea_priv->pipeA_y = y; - break; - case 1: - master_priv->sarea_priv->pipeB_x = x; - master_priv->sarea_priv->pipeB_y = y; - break; - default: - break; - } + return pending; } static int @@ -2336,6 +2650,11 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, struct drm_framebuffer *old_fb; int ret; + if (intel_crtc_has_pending_flip(crtc)) { + DRM_ERROR("pipe is still busy with an old pageflip\n"); + return -EBUSY; + } + /* no fb bound */ if (!fb) { DRM_ERROR("No FB bound\n"); @@ -2353,8 +2672,8 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, ret = intel_pin_and_fence_fb_obj(dev, to_intel_framebuffer(fb)->obj, NULL); + mutex_unlock(&dev->struct_mutex); if (ret != 0) { - mutex_unlock(&dev->struct_mutex); DRM_ERROR("pin & fence failed\n"); return ret; } @@ -2372,7 +2691,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, * whether the platform allows pfit disable with pipe active, and only * then update the pipesrc and pfit state, even on the flip path. */ - if (i915_fastboot) { + if (i915.fastboot) { const struct drm_display_mode *adjusted_mode = &intel_crtc->config.adjusted_mode; @@ -2390,31 +2709,26 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, intel_crtc->config.pipe_src_h = adjusted_mode->crtc_vdisplay; } - ret = dev_priv->display.update_plane(crtc, fb, x, y); - if (ret) { - intel_unpin_fb_obj(to_intel_framebuffer(fb)->obj); - mutex_unlock(&dev->struct_mutex); - DRM_ERROR("failed to update base address\n"); - return ret; - } + dev_priv->display.update_primary_plane(crtc, fb, x, y); - old_fb = crtc->fb; - crtc->fb = fb; + old_fb = crtc->primary->fb; + crtc->primary->fb = fb; crtc->x = x; crtc->y = y; if (old_fb) { if (intel_crtc->active && old_fb != fb) intel_wait_for_vblank(dev, intel_crtc->pipe); + mutex_lock(&dev->struct_mutex); intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj); + mutex_unlock(&dev->struct_mutex); } + mutex_lock(&dev->struct_mutex); intel_update_fbc(dev); intel_edp_psr_update(dev); mutex_unlock(&dev->struct_mutex); - intel_crtc_update_sarea_pos(crtc, x, y); - return 0; } @@ -2498,12 +2812,10 @@ static void ironlake_fdi_link_train(struct drm_crtc *crtc) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; u32 reg, temp, tries; - /* FDI needs bits from pipe & plane first */ + /* FDI needs bits from pipe first */ assert_pipe_enabled(dev_priv, pipe); - assert_plane_enabled(dev_priv, plane); /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit for train result */ @@ -2934,9 +3246,8 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc) udelay(100); /* Ironlake workaround, disable clock pointer after downing FDI */ - if (HAS_PCH_IBX(dev)) { + if (HAS_PCH_IBX(dev)) I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR); - } /* still set train pattern 1 */ reg = FDI_TX_CTL(pipe); @@ -2963,25 +3274,6 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc) udelay(100); } -static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - unsigned long flags; - bool pending; - - if (i915_reset_in_progress(&dev_priv->gpu_error) || - intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) - return false; - - spin_lock_irqsave(&dev->event_lock, flags); - pending = to_intel_crtc(crtc)->unpin_work != NULL; - spin_unlock_irqrestore(&dev->event_lock, flags); - - return pending; -} - bool intel_has_pending_fb_unpin(struct drm_device *dev) { struct intel_crtc *crtc; @@ -2993,7 +3285,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) * cannot claim and pin a new fb without at least acquring the * struct_mutex and so serialising with us. */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) { + for_each_intel_crtc(dev, crtc) { if (atomic_read(&crtc->unpin_work_count) == 0) continue; @@ -3006,21 +3298,22 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) return false; } -static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) +void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - if (crtc->fb == NULL) + if (crtc->primary->fb == NULL) return; WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue)); - wait_event(dev_priv->pending_flip_queue, - !intel_crtc_has_pending_flip(crtc)); + WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue, + !intel_crtc_has_pending_flip(crtc), + 60*HZ) == 0); mutex_lock(&dev->struct_mutex); - intel_finish_fb(crtc->fb); + intel_finish_fb(crtc->primary->fb); mutex_unlock(&dev->struct_mutex); } @@ -3230,7 +3523,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) * Note that enable_shared_dpll tries to do the right thing, but * get_shared_dpll unconditionally resets the pll - we need that to have * the right LVDS enable sequence. */ - ironlake_enable_shared_dpll(intel_crtc); + intel_enable_shared_dpll(intel_crtc); /* set transcoder timing, panel must allow it */ assert_panel_unlocked(dev_priv, pipe); @@ -3334,6 +3627,8 @@ static struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc) DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n", crtc->base.base.id, pll->name); + WARN_ON(pll->refcount); + goto found; } @@ -3367,20 +3662,13 @@ static struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc) return NULL; found: + if (pll->refcount == 0) + pll->hw_state = crtc->config.dpll_hw_state; + crtc->config.shared_dpll = i; DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name, pipe_name(crtc->pipe)); - if (pll->active == 0) { - memcpy(&pll->hw_state, &crtc->config.dpll_hw_state, - sizeof(pll->hw_state)); - - DRM_DEBUG_DRIVER("setting up %s\n", pll->name); - WARN_ON(pll->on); - assert_shared_dpll_disabled(dev_priv, pll); - - pll->mode_set(dev_priv, pll); - } pll->refcount++; return pll; @@ -3425,37 +3713,43 @@ static void intel_enable_planes(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; enum pipe pipe = to_intel_crtc(crtc)->pipe; + struct drm_plane *plane; struct intel_plane *intel_plane; - list_for_each_entry(intel_plane, &dev->mode_config.plane_list, base.head) + drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { + intel_plane = to_intel_plane(plane); if (intel_plane->pipe == pipe) intel_plane_restore(&intel_plane->base); + } } static void intel_disable_planes(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; enum pipe pipe = to_intel_crtc(crtc)->pipe; + struct drm_plane *plane; struct intel_plane *intel_plane; - list_for_each_entry(intel_plane, &dev->mode_config.plane_list, base.head) + drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { + intel_plane = to_intel_plane(plane); if (intel_plane->pipe == pipe) intel_plane_disable(&intel_plane->base); + } } void hsw_enable_ips(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; if (!crtc->config.ips_enabled) return; - /* We can only enable IPS after we enable a plane and wait for a vblank. - * We guarantee that the plane is enabled by calling intel_enable_ips - * only after intel_enable_plane. And intel_enable_plane already waits - * for a vblank, so all we need to do here is to enable the IPS bit. */ + /* We can only enable IPS after we enable a plane and wait for a vblank */ + intel_wait_for_vblank(dev, crtc->pipe); + assert_plane_enabled(dev_priv, crtc->plane); - if (IS_BROADWELL(crtc->base.dev)) { + if (IS_BROADWELL(dev)) { mutex_lock(&dev_priv->rps.hw_lock); WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000)); mutex_unlock(&dev_priv->rps.hw_lock); @@ -3485,10 +3779,13 @@ void hsw_disable_ips(struct intel_crtc *crtc) return; assert_plane_enabled(dev_priv, crtc->plane); - if (IS_BROADWELL(crtc->base.dev)) { + if (IS_BROADWELL(dev)) { mutex_lock(&dev_priv->rps.hw_lock); WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0)); mutex_unlock(&dev_priv->rps.hw_lock); + /* wait for pcode to finish disabling IPS, which may take up to 42ms */ + if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42)) + DRM_ERROR("Timed out waiting for IPS disable\n"); } else { I915_WRITE(IPS_CTL, 0); POSTING_READ(IPS_CTL); @@ -3545,6 +3842,94 @@ static void intel_crtc_load_lut(struct drm_crtc *crtc) hsw_enable_ips(intel_crtc); } +static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable) +{ + if (!enable && intel_crtc->overlay) { + struct drm_device *dev = intel_crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + mutex_lock(&dev->struct_mutex); + dev_priv->mm.interruptible = false; + (void) intel_overlay_switch_off(intel_crtc->overlay); + dev_priv->mm.interruptible = true; + mutex_unlock(&dev->struct_mutex); + } + + /* Let userspace switch the overlay on again. In most cases userspace + * has to recompute where to put it anyway. + */ +} + +/** + * i9xx_fixup_plane - ugly workaround for G45 to fire up the hardware + * cursor plane briefly if not already running after enabling the display + * plane. + * This workaround avoids occasional blank screens when self refresh is + * enabled. + */ +static void +g4x_fixup_plane(struct drm_i915_private *dev_priv, enum pipe pipe) +{ + u32 cntl = I915_READ(CURCNTR(pipe)); + + if ((cntl & CURSOR_MODE) == 0) { + u32 fw_bcl_self = I915_READ(FW_BLC_SELF); + + I915_WRITE(FW_BLC_SELF, fw_bcl_self & ~FW_BLC_SELF_EN); + I915_WRITE(CURCNTR(pipe), CURSOR_MODE_64_ARGB_AX); + intel_wait_for_vblank(dev_priv->dev, pipe); + I915_WRITE(CURCNTR(pipe), cntl); + I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe))); + I915_WRITE(FW_BLC_SELF, fw_bcl_self); + } +} + +static void intel_crtc_enable_planes(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + int pipe = intel_crtc->pipe; + int plane = intel_crtc->plane; + + intel_enable_primary_hw_plane(dev_priv, plane, pipe); + intel_enable_planes(crtc); + /* The fixup needs to happen before cursor is enabled */ + if (IS_G4X(dev)) + g4x_fixup_plane(dev_priv, pipe); + intel_crtc_update_cursor(crtc, true); + intel_crtc_dpms_overlay(intel_crtc, true); + + hsw_enable_ips(intel_crtc); + + mutex_lock(&dev->struct_mutex); + intel_update_fbc(dev); + intel_edp_psr_update(dev); + mutex_unlock(&dev->struct_mutex); +} + +static void intel_crtc_disable_planes(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + int pipe = intel_crtc->pipe; + int plane = intel_crtc->plane; + + intel_crtc_wait_for_pending_flips(crtc); + drm_crtc_vblank_off(crtc); + + if (dev_priv->fbc.plane == plane) + intel_disable_fbc(dev); + + hsw_disable_ips(intel_crtc); + + intel_crtc_dpms_overlay(intel_crtc, false); + intel_crtc_update_cursor(crtc, false); + intel_disable_planes(crtc); + intel_disable_primary_hw_plane(dev_priv, plane, pipe); +} + static void ironlake_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; @@ -3552,13 +3937,35 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; + enum plane plane = intel_crtc->plane; WARN_ON(!crtc->enabled); if (intel_crtc->active) return; + if (intel_crtc->config.has_pch_encoder) + intel_prepare_shared_dpll(intel_crtc); + + if (intel_crtc->config.has_dp_encoder) + intel_dp_set_m_n(intel_crtc); + + intel_set_pipe_timings(intel_crtc); + + if (intel_crtc->config.has_pch_encoder) { + intel_cpu_transcoder_set_m_n(intel_crtc, + &intel_crtc->config.fdi_m_n); + } + + ironlake_set_pipeconf(crtc); + + /* Set up the display plane register */ + I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE); + POSTING_READ(DSPCNTR(plane)); + + dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, + crtc->x, crtc->y); + intel_crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); @@ -3587,34 +3994,20 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) intel_crtc_load_lut(crtc); intel_update_watermarks(crtc); - intel_enable_pipe(dev_priv, pipe, - intel_crtc->config.has_pch_encoder, false); - intel_enable_primary_plane(dev_priv, plane, pipe); - intel_enable_planes(crtc); - intel_crtc_update_cursor(crtc, true); + intel_enable_pipe(intel_crtc); if (intel_crtc->config.has_pch_encoder) ironlake_pch_enable(crtc); - mutex_lock(&dev->struct_mutex); - intel_update_fbc(dev); - mutex_unlock(&dev->struct_mutex); - for_each_encoder_on_crtc(dev, crtc, encoder) encoder->enable(encoder); if (HAS_PCH_CPT(dev)) cpt_verify_modeset(dev, intel_crtc->pipe); - /* - * There seems to be a race in PCH platform hw (at least on some - * outputs) where an enabled pipe still completes any pageflip right - * away (as if the pipe is off) instead of waiting for vblank. As soon - * as the first vblank happend, everything works as expected. Hence just - * wait for one vblank before returning to avoid strange things - * happening. - */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_crtc_enable_planes(crtc); + + drm_crtc_vblank_on(crtc); } /* IPS only exists on ULT machines and is tied to pipe A. */ @@ -3623,47 +4016,6 @@ static bool hsw_crtc_supports_ips(struct intel_crtc *crtc) return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A; } -static void haswell_crtc_enable_planes(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; - - intel_enable_primary_plane(dev_priv, plane, pipe); - intel_enable_planes(crtc); - intel_crtc_update_cursor(crtc, true); - - hsw_enable_ips(intel_crtc); - - mutex_lock(&dev->struct_mutex); - intel_update_fbc(dev); - mutex_unlock(&dev->struct_mutex); -} - -static void haswell_crtc_disable_planes(struct drm_crtc *crtc) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; - - intel_crtc_wait_for_pending_flips(crtc); - drm_vblank_off(dev, pipe); - - /* FBC must be disabled before disabling the plane on HSW. */ - if (dev_priv->fbc.plane == plane) - intel_disable_fbc(dev); - - hsw_disable_ips(intel_crtc); - - intel_crtc_update_cursor(crtc, false); - intel_disable_planes(crtc); - intel_disable_primary_plane(dev_priv, plane, pipe); -} - /* * This implements the workaround described in the "notes" section of the mode * set sequence documentation. When going from no pipes or single pipe to @@ -3677,7 +4029,7 @@ static void haswell_mode_set_planes_workaround(struct intel_crtc *crtc) /* We want to get the other_active_crtc only if there's only 1 other * active crtc. */ - list_for_each_entry(crtc_it, &dev->mode_config.crtc_list, base.head) { + for_each_intel_crtc(dev, crtc_it) { if (!crtc_it->active || crtc_it == crtc) continue; @@ -3700,12 +4052,34 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; + enum plane plane = intel_crtc->plane; WARN_ON(!crtc->enabled); if (intel_crtc->active) return; + if (intel_crtc->config.has_dp_encoder) + intel_dp_set_m_n(intel_crtc); + + intel_set_pipe_timings(intel_crtc); + + if (intel_crtc->config.has_pch_encoder) { + intel_cpu_transcoder_set_m_n(intel_crtc, + &intel_crtc->config.fdi_m_n); + } + + haswell_set_pipeconf(crtc); + + intel_set_pipe_csc(crtc); + + /* Set up the display plane register */ + I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE | DISPPLANE_PIPE_CSC_ENABLE); + POSTING_READ(DSPCNTR(plane)); + + dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, + crtc->x, crtc->y); + intel_crtc->active = true; intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); @@ -3733,8 +4107,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_ddi_enable_transcoder_func(crtc); intel_update_watermarks(crtc); - intel_enable_pipe(dev_priv, pipe, - intel_crtc->config.has_pch_encoder, false); + intel_enable_pipe(intel_crtc); if (intel_crtc->config.has_pch_encoder) lpt_pch_enable(crtc); @@ -3747,17 +4120,9 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) /* If we change the relative order between pipe/planes enabling, we need * to change the workaround. */ haswell_mode_set_planes_workaround(intel_crtc); - haswell_crtc_enable_planes(crtc); + intel_crtc_enable_planes(crtc); - /* - * There seems to be a race in PCH platform hw (at least on some - * outputs) where an enabled pipe still completes any pageflip right - * away (as if the pipe is off) instead of waiting for vblank. As soon - * as the first vblank happend, everything works as expected. Hence just - * wait for one vblank before returning to avoid strange things - * happening. - */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + drm_crtc_vblank_on(crtc); } static void ironlake_pfit_disable(struct intel_crtc *crtc) @@ -3782,26 +4147,16 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; u32 reg, temp; - if (!intel_crtc->active) return; + intel_crtc_disable_planes(crtc); + for_each_encoder_on_crtc(dev, crtc, encoder) encoder->disable(encoder); - intel_crtc_wait_for_pending_flips(crtc); - drm_vblank_off(dev, pipe); - - if (dev_priv->fbc.plane == plane) - intel_disable_fbc(dev); - - intel_crtc_update_cursor(crtc, false); - intel_disable_planes(crtc); - intel_disable_primary_plane(dev_priv, plane, pipe); - if (intel_crtc->config.has_pch_encoder) intel_set_pch_fifo_underrun_reporting(dev, pipe, false); @@ -3845,6 +4200,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) mutex_lock(&dev->struct_mutex); intel_update_fbc(dev); + intel_edp_psr_update(dev); mutex_unlock(&dev->struct_mutex); } @@ -3860,7 +4216,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) if (!intel_crtc->active) return; - haswell_crtc_disable_planes(crtc); + intel_crtc_disable_planes(crtc); for_each_encoder_on_crtc(dev, crtc, encoder) { intel_opregion_notify_encoder(encoder, false); @@ -3892,6 +4248,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) mutex_lock(&dev->struct_mutex); intel_update_fbc(dev); + intel_edp_psr_update(dev); mutex_unlock(&dev->struct_mutex); } @@ -3906,48 +4263,6 @@ static void haswell_crtc_off(struct drm_crtc *crtc) intel_ddi_put_crtc_pll(crtc); } -static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable) -{ - if (!enable && intel_crtc->overlay) { - struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - - mutex_lock(&dev->struct_mutex); - dev_priv->mm.interruptible = false; - (void) intel_overlay_switch_off(intel_crtc->overlay); - dev_priv->mm.interruptible = true; - mutex_unlock(&dev->struct_mutex); - } - - /* Let userspace switch the overlay on again. In most cases userspace - * has to recompute where to put it anyway. - */ -} - -/** - * i9xx_fixup_plane - ugly workaround for G45 to fire up the hardware - * cursor plane briefly if not already running after enabling the display - * plane. - * This workaround avoids occasional blank screens when self refresh is - * enabled. - */ -static void -g4x_fixup_plane(struct drm_i915_private *dev_priv, enum pipe pipe) -{ - u32 cntl = I915_READ(CURCNTR(pipe)); - - if ((cntl & CURSOR_MODE) == 0) { - u32 fw_bcl_self = I915_READ(FW_BLC_SELF); - - I915_WRITE(FW_BLC_SELF, fw_bcl_self & ~FW_BLC_SELF_EN); - I915_WRITE(CURCNTR(pipe), CURSOR_MODE_64_ARGB_AX); - intel_wait_for_vblank(dev_priv->dev, pipe); - I915_WRITE(CURCNTR(pipe), cntl); - I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe))); - I915_WRITE(FW_BLC_SELF, fw_bcl_self); - } -} - static void i9xx_pfit_enable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -3972,6 +4287,117 @@ static void i9xx_pfit_enable(struct intel_crtc *crtc) I915_WRITE(BCLRPAT(crtc->pipe), 0); } +#define for_each_power_domain(domain, mask) \ + for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ + if ((1 << (domain)) & (mask)) + +enum intel_display_power_domain +intel_display_port_power_domain(struct intel_encoder *intel_encoder) +{ + struct drm_device *dev = intel_encoder->base.dev; + struct intel_digital_port *intel_dig_port; + + switch (intel_encoder->type) { + case INTEL_OUTPUT_UNKNOWN: + /* Only DDI platforms should ever use this output type */ + WARN_ON_ONCE(!HAS_DDI(dev)); + case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_HDMI: + case INTEL_OUTPUT_EDP: + intel_dig_port = enc_to_dig_port(&intel_encoder->base); + switch (intel_dig_port->port) { + case PORT_A: + return POWER_DOMAIN_PORT_DDI_A_4_LANES; + case PORT_B: + return POWER_DOMAIN_PORT_DDI_B_4_LANES; + case PORT_C: + return POWER_DOMAIN_PORT_DDI_C_4_LANES; + case PORT_D: + return POWER_DOMAIN_PORT_DDI_D_4_LANES; + default: + WARN_ON_ONCE(1); + return POWER_DOMAIN_PORT_OTHER; + } + case INTEL_OUTPUT_ANALOG: + return POWER_DOMAIN_PORT_CRT; + case INTEL_OUTPUT_DSI: + return POWER_DOMAIN_PORT_DSI; + default: + return POWER_DOMAIN_PORT_OTHER; + } +} + +static unsigned long get_crtc_power_domains(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct intel_encoder *intel_encoder; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + enum pipe pipe = intel_crtc->pipe; + bool pfit_enabled = intel_crtc->config.pch_pfit.enabled; + unsigned long mask; + enum transcoder transcoder; + + transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe); + + mask = BIT(POWER_DOMAIN_PIPE(pipe)); + mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder)); + if (pfit_enabled) + mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe)); + + for_each_encoder_on_crtc(dev, crtc, intel_encoder) + mask |= BIT(intel_display_port_power_domain(intel_encoder)); + + return mask; +} + +void intel_display_set_init_power(struct drm_i915_private *dev_priv, + bool enable) +{ + if (dev_priv->power_domains.init_power_on == enable) + return; + + if (enable) + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + else + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + + dev_priv->power_domains.init_power_on = enable; +} + +static void modeset_update_crtc_power_domains(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long pipe_domains[I915_MAX_PIPES] = { 0, }; + struct intel_crtc *crtc; + + /* + * First get all needed power domains, then put all unneeded, to avoid + * any unnecessary toggling of the power wells. + */ + for_each_intel_crtc(dev, crtc) { + enum intel_display_power_domain domain; + + if (!crtc->base.enabled) + continue; + + pipe_domains[crtc->pipe] = get_crtc_power_domains(&crtc->base); + + for_each_power_domain(domain, pipe_domains[crtc->pipe]) + intel_display_power_get(dev_priv, domain); + } + + for_each_intel_crtc(dev, crtc) { + enum intel_display_power_domain domain; + + for_each_power_domain(domain, crtc->enabled_power_domains) + intel_display_power_put(dev_priv, domain); + + crtc->enabled_power_domains = pipe_domains[crtc->pipe]; + } + + intel_display_set_init_power(dev_priv, false); +} + int valleyview_get_vco(struct drm_i915_private *dev_priv) { int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 }; @@ -3991,6 +4417,9 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) struct drm_i915_private *dev_priv = dev->dev_private; u32 val, cmd; + WARN_ON(valleyview_cur_cdclk(dev_priv) != dev_priv->vlv_cdclk_freq); + dev_priv->vlv_cdclk_freq = cdclk; + if (cdclk >= 320) /* jump to highest voltage for 400MHz too */ cmd = 2; else if (cdclk == 266) @@ -4045,7 +4474,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) intel_i2c_reset(dev); } -static int valleyview_cur_cdclk(struct drm_i915_private *dev_priv) +int valleyview_cur_cdclk(struct drm_i915_private *dev_priv) { int cur_cdclk, vco; int divider; @@ -4066,10 +4495,6 @@ static int valleyview_cur_cdclk(struct drm_i915_private *dev_priv) static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, int max_pixclk) { - int cur_cdclk; - - cur_cdclk = valleyview_cur_cdclk(dev_priv); - /* * Really only a few cases to deal with, as only 4 CDclks are supported: * 200MHz @@ -4088,43 +4513,35 @@ static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, /* Looks like the 200MHz CDclk freq doesn't work on some configs */ } -static int intel_mode_max_pixclk(struct drm_i915_private *dev_priv, - unsigned modeset_pipes, - struct intel_crtc_config *pipe_config) +/* compute the max pixel clock for new configuration */ +static int intel_mode_max_pixclk(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; struct intel_crtc *intel_crtc; int max_pixclk = 0; - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, - base.head) { - if (modeset_pipes & (1 << intel_crtc->pipe)) - max_pixclk = max(max_pixclk, - pipe_config->adjusted_mode.crtc_clock); - else if (intel_crtc->base.enabled) + for_each_intel_crtc(dev, intel_crtc) { + if (intel_crtc->new_enabled) max_pixclk = max(max_pixclk, - intel_crtc->config.adjusted_mode.crtc_clock); + intel_crtc->new_config->adjusted_mode.crtc_clock); } return max_pixclk; } static void valleyview_modeset_global_pipes(struct drm_device *dev, - unsigned *prepare_pipes, - unsigned modeset_pipes, - struct intel_crtc_config *pipe_config) + unsigned *prepare_pipes) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc; - int max_pixclk = intel_mode_max_pixclk(dev_priv, modeset_pipes, - pipe_config); - int cur_cdclk = valleyview_cur_cdclk(dev_priv); + int max_pixclk = intel_mode_max_pixclk(dev_priv); - if (valleyview_calc_cdclk(dev_priv, max_pixclk) == cur_cdclk) + if (valleyview_calc_cdclk(dev_priv, max_pixclk) == + dev_priv->vlv_cdclk_freq) return; - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, - base.head) + /* disable/enable all currently active pipes while we change cdclk */ + for_each_intel_crtc(dev, intel_crtc) if (intel_crtc->base.enabled) *prepare_pipes |= (1 << intel_crtc->pipe); } @@ -4132,12 +4549,12 @@ static void valleyview_modeset_global_pipes(struct drm_device *dev, static void valleyview_modeset_global_resources(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - int max_pixclk = intel_mode_max_pixclk(dev_priv, 0, NULL); - int cur_cdclk = valleyview_cur_cdclk(dev_priv); + int max_pixclk = intel_mode_max_pixclk(dev_priv); int req_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk); - if (req_cdclk != cur_cdclk) + if (req_cdclk != dev_priv->vlv_cdclk_freq) valleyview_set_cdclk(dev, req_cdclk); + modeset_update_crtc_power_domains(dev); } static void valleyview_crtc_enable(struct drm_crtc *crtc) @@ -4149,22 +4566,56 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; bool is_dsi; + u32 dspcntr; WARN_ON(!crtc->enabled); if (intel_crtc->active) return; + is_dsi = intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI); + + if (!is_dsi && !IS_CHERRYVIEW(dev)) + vlv_prepare_pll(intel_crtc); + + /* Set up the display plane register */ + dspcntr = DISPPLANE_GAMMA_ENABLE; + + if (intel_crtc->config.has_dp_encoder) + intel_dp_set_m_n(intel_crtc); + + intel_set_pipe_timings(intel_crtc); + + /* pipesrc and dspsize control the size that is scaled from, + * which should always be the user's requested size. + */ + I915_WRITE(DSPSIZE(plane), + ((intel_crtc->config.pipe_src_h - 1) << 16) | + (intel_crtc->config.pipe_src_w - 1)); + I915_WRITE(DSPPOS(plane), 0); + + i9xx_set_pipeconf(intel_crtc); + + I915_WRITE(DSPCNTR(plane), dspcntr); + POSTING_READ(DSPCNTR(plane)); + + dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, + crtc->x, crtc->y); + intel_crtc->active = true; + intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); + for_each_encoder_on_crtc(dev, crtc, encoder) if (encoder->pre_pll_enable) encoder->pre_pll_enable(encoder); - is_dsi = intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI); - - if (!is_dsi) - vlv_enable_pll(intel_crtc); + if (!is_dsi) { + if (IS_CHERRYVIEW(dev)) + chv_enable_pll(intel_crtc); + else + vlv_enable_pll(intel_crtc); + } for_each_encoder_on_crtc(dev, crtc, encoder) if (encoder->pre_enable) @@ -4175,15 +4626,26 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) intel_crtc_load_lut(crtc); intel_update_watermarks(crtc); - intel_enable_pipe(dev_priv, pipe, false, is_dsi); - intel_enable_primary_plane(dev_priv, plane, pipe); - intel_enable_planes(crtc); - intel_crtc_update_cursor(crtc, true); - - intel_update_fbc(dev); + intel_enable_pipe(intel_crtc); for_each_encoder_on_crtc(dev, crtc, encoder) encoder->enable(encoder); + + intel_crtc_enable_planes(crtc); + + drm_crtc_vblank_on(crtc); + + /* Underruns don't raise interrupts, so check manually. */ + i9xx_check_fifo_underruns(dev); +} + +static void i9xx_set_pll_dividers(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(FP0(crtc->pipe), crtc->config.dpll_hw_state.fp0); + I915_WRITE(FP1(crtc->pipe), crtc->config.dpll_hw_state.fp1); } static void i9xx_crtc_enable(struct drm_crtc *crtc) @@ -4194,14 +4656,49 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) struct intel_encoder *encoder; int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; + u32 dspcntr; WARN_ON(!crtc->enabled); if (intel_crtc->active) return; + i9xx_set_pll_dividers(intel_crtc); + + /* Set up the display plane register */ + dspcntr = DISPPLANE_GAMMA_ENABLE; + + if (pipe == 0) + dspcntr &= ~DISPPLANE_SEL_PIPE_MASK; + else + dspcntr |= DISPPLANE_SEL_PIPE_B; + + if (intel_crtc->config.has_dp_encoder) + intel_dp_set_m_n(intel_crtc); + + intel_set_pipe_timings(intel_crtc); + + /* pipesrc and dspsize control the size that is scaled from, + * which should always be the user's requested size. + */ + I915_WRITE(DSPSIZE(plane), + ((intel_crtc->config.pipe_src_h - 1) << 16) | + (intel_crtc->config.pipe_src_w - 1)); + I915_WRITE(DSPPOS(plane), 0); + + i9xx_set_pipeconf(intel_crtc); + + I915_WRITE(DSPCNTR(plane), dspcntr); + POSTING_READ(DSPCNTR(plane)); + + dev_priv->display.update_primary_plane(crtc, crtc->primary->fb, + crtc->x, crtc->y); + intel_crtc->active = true; + if (!IS_GEN2(dev)) + intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); + for_each_encoder_on_crtc(dev, crtc, encoder) if (encoder->pre_enable) encoder->pre_enable(encoder); @@ -4213,21 +4710,27 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) intel_crtc_load_lut(crtc); intel_update_watermarks(crtc); - intel_enable_pipe(dev_priv, pipe, false, false); - intel_enable_primary_plane(dev_priv, plane, pipe); - intel_enable_planes(crtc); - /* The fixup needs to happen before cursor is enabled */ - if (IS_G4X(dev)) - g4x_fixup_plane(dev_priv, pipe); - intel_crtc_update_cursor(crtc, true); - - /* Give the overlay scaler a chance to enable if it's on this pipe */ - intel_crtc_dpms_overlay(intel_crtc, true); - - intel_update_fbc(dev); + intel_enable_pipe(intel_crtc); for_each_encoder_on_crtc(dev, crtc, encoder) encoder->enable(encoder); + + intel_crtc_enable_planes(crtc); + + /* + * Gen2 reports pipe underruns whenever all planes are disabled. + * So don't enable underrun reporting before at least some planes + * are enabled. + * FIXME: Need to fix the logic to work when we turn off all planes + * but leave the pipe running. + */ + if (IS_GEN2(dev)) + intel_set_cpu_fifo_underrun_reporting(dev, pipe, true); + + drm_crtc_vblank_on(crtc); + + /* Underruns don't raise interrupts, so check manually. */ + i9xx_check_fifo_underruns(dev); } static void i9xx_pfit_disable(struct intel_crtc *crtc) @@ -4252,25 +4755,30 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; if (!intel_crtc->active) return; - for_each_encoder_on_crtc(dev, crtc, encoder) - encoder->disable(encoder); + /* + * Gen2 reports pipe underruns whenever all planes are disabled. + * So diasble underrun reporting before all the planes get disabled. + * FIXME: Need to fix the logic to work when we turn off all planes + * but leave the pipe running. + */ + if (IS_GEN2(dev)) + intel_set_cpu_fifo_underrun_reporting(dev, pipe, false); - /* Give the overlay scaler a chance to disable if it's on this pipe */ - intel_crtc_wait_for_pending_flips(crtc); - drm_vblank_off(dev, pipe); + intel_crtc_disable_planes(crtc); - if (dev_priv->fbc.plane == plane) - intel_disable_fbc(dev); + for_each_encoder_on_crtc(dev, crtc, encoder) + encoder->disable(encoder); - intel_crtc_dpms_overlay(intel_crtc, false); - intel_crtc_update_cursor(crtc, false); - intel_disable_planes(crtc); - intel_disable_primary_plane(dev_priv, plane, pipe); + /* + * On gen2 planes are double buffered but the pipe isn't, so we must + * wait for planes to fully turn off before disabling the pipe. + */ + if (IS_GEN2(dev)) + intel_wait_for_vblank(dev, pipe); intel_disable_pipe(dev_priv, pipe); @@ -4280,15 +4788,25 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) if (encoder->post_disable) encoder->post_disable(encoder); - if (IS_VALLEYVIEW(dev) && !intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI)) - vlv_disable_pll(dev_priv, pipe); - else if (!IS_VALLEYVIEW(dev)) - i9xx_disable_pll(dev_priv, pipe); + if (!intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI)) { + if (IS_CHERRYVIEW(dev)) + chv_disable_pll(dev_priv, pipe); + else if (IS_VALLEYVIEW(dev)) + vlv_disable_pll(dev_priv, pipe); + else + i9xx_disable_pll(dev_priv, pipe); + } + + if (!IS_GEN2(dev)) + intel_set_cpu_fifo_underrun_reporting(dev, pipe, false); intel_crtc->active = false; intel_update_watermarks(crtc); + mutex_lock(&dev->struct_mutex); intel_update_fbc(dev); + intel_edp_psr_update(dev); + mutex_unlock(&dev->struct_mutex); } static void i9xx_crtc_off(struct drm_crtc *crtc) @@ -4351,13 +4869,11 @@ static void intel_crtc_disable(struct drm_crtc *crtc) struct drm_device *dev = crtc->dev; struct drm_connector *connector; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); /* crtc should still be enabled when we disable it. */ WARN_ON(!crtc->enabled); dev_priv->display.crtc_disable(crtc); - intel_crtc->eld_vld = false; intel_crtc_update_sarea(crtc, false); dev_priv->display.off(crtc); @@ -4365,11 +4881,11 @@ static void intel_crtc_disable(struct drm_crtc *crtc) assert_cursor_disabled(dev_priv, to_intel_crtc(crtc)->pipe); assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe); - if (crtc->fb) { + if (crtc->primary->fb) { mutex_lock(&dev->struct_mutex); - intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj); + intel_unpin_fb_obj(to_intel_framebuffer(crtc->primary->fb)->obj); mutex_unlock(&dev->struct_mutex); - crtc->fb = NULL; + crtc->primary->fb = NULL; } /* Update computed state. */ @@ -4421,7 +4937,7 @@ static void intel_connector_check_state(struct intel_connector *connector) DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.base.id, - drm_get_connector_name(&connector->base)); + connector->base.name); WARN(connector->base.dpms == DRM_MODE_DPMS_OFF, "wrong connector dpms state\n"); @@ -4583,7 +5099,7 @@ retry: static void hsw_compute_ips_config(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config) { - pipe_config->ips_enabled = i915_enable_ips && + pipe_config->ips_enabled = i915.enable_ips && hsw_crtc_supports_ips(crtc) && pipe_config->pipe_bpp <= 24; } @@ -4784,8 +5300,8 @@ intel_link_compute_m_n(int bits_per_pixel, int nlanes, static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) { - if (i915_panel_use_ssc >= 0) - return i915_panel_use_ssc != 0; + if (i915.panel_use_ssc >= 0) + return i915.panel_use_ssc != 0; return dev_priv->vbt.lvds_use_ssc && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE); } @@ -4825,8 +5341,6 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc, intel_clock_t *reduced_clock) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int pipe = crtc->pipe; u32 fp, fp2 = 0; if (IS_PINEVIEW(dev)) { @@ -4839,17 +5353,14 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc, fp2 = i9xx_dpll_compute_fp(reduced_clock); } - I915_WRITE(FP0(pipe), fp); crtc->config.dpll_hw_state.fp0 = fp; crtc->lowfreq_avail = false; if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) && - reduced_clock && i915_powersave) { - I915_WRITE(FP1(pipe), fp2); + reduced_clock && i915.powersave) { crtc->config.dpll_hw_state.fp1 = fp2; crtc->lowfreq_avail = true; } else { - I915_WRITE(FP1(pipe), fp); crtc->config.dpll_hw_state.fp1 = fp; } } @@ -4927,12 +5438,34 @@ static void intel_dp_set_m_n(struct intel_crtc *crtc) static void vlv_update_pll(struct intel_crtc *crtc) { + u32 dpll, dpll_md; + + /* + * Enable DPIO clock input. We should never disable the reference + * clock for pipe B, since VGA hotplug / manual detection depends + * on it. + */ + dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REFA_CLK_ENABLE_VLV | + DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_CLOCK_VLV; + /* We should never disable this, set it here for state tracking */ + if (crtc->pipe == PIPE_B) + dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; + dpll |= DPLL_VCO_ENABLE; + crtc->config.dpll_hw_state.dpll = dpll; + + dpll_md = (crtc->config.pixel_multiplier - 1) + << DPLL_MD_UDI_MULTIPLIER_SHIFT; + crtc->config.dpll_hw_state.dpll_md = dpll_md; +} + +static void vlv_prepare_pll(struct intel_crtc *crtc) +{ struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int pipe = crtc->pipe; - u32 dpll, mdiv; + u32 mdiv; u32 bestn, bestm1, bestm2, bestp1, bestp2; - u32 coreclk, reg_val, dpll_md; + u32 coreclk, reg_val; mutex_lock(&dev_priv->dpio_lock); @@ -4945,7 +5478,7 @@ static void vlv_update_pll(struct intel_crtc *crtc) /* See eDP HDMI DPIO driver vbios notes doc */ /* PLL B needs special handling */ - if (pipe) + if (pipe == PIPE_B) vlv_pllb_recal_opamp(dev_priv, pipe); /* Set up Tx target for periodic Rcomp update */ @@ -4989,7 +5522,7 @@ static void vlv_update_pll(struct intel_crtc *crtc) if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_EDP) || intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DISPLAYPORT)) { /* Use SSC source */ - if (!pipe) + if (pipe == PIPE_A) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), 0x0df40000); else @@ -4997,7 +5530,7 @@ static void vlv_update_pll(struct intel_crtc *crtc) 0x0df70000); } else { /* HDMI or VGA */ /* Use bend source */ - if (!pipe) + if (pipe == PIPE_A) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), 0x0df70000); else @@ -5013,26 +5546,84 @@ static void vlv_update_pll(struct intel_crtc *crtc) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk); vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000); + mutex_unlock(&dev_priv->dpio_lock); +} + +static void chv_update_pll(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe = crtc->pipe; + int dpll_reg = DPLL(crtc->pipe); + enum dpio_channel port = vlv_pipe_to_channel(pipe); + u32 loopfilter, intcoeff; + u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac; + int refclk; + + crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV | + DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS | + DPLL_VCO_ENABLE; + if (pipe != PIPE_A) + crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; + + crtc->config.dpll_hw_state.dpll_md = + (crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; + + bestn = crtc->config.dpll.n; + bestm2_frac = crtc->config.dpll.m2 & 0x3fffff; + bestm1 = crtc->config.dpll.m1; + bestm2 = crtc->config.dpll.m2 >> 22; + bestp1 = crtc->config.dpll.p1; + bestp2 = crtc->config.dpll.p2; /* - * Enable DPIO clock input. We should never disable the reference - * clock for pipe B, since VGA hotplug / manual detection depends - * on it. + * Enable Refclk and SSC */ - dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REFA_CLK_ENABLE_VLV | - DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_CLOCK_VLV; - /* We should never disable this, set it here for state tracking */ - if (pipe == PIPE_B) - dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; - dpll |= DPLL_VCO_ENABLE; - crtc->config.dpll_hw_state.dpll = dpll; + I915_WRITE(dpll_reg, + crtc->config.dpll_hw_state.dpll & ~DPLL_VCO_ENABLE); - dpll_md = (crtc->config.pixel_multiplier - 1) - << DPLL_MD_UDI_MULTIPLIER_SHIFT; - crtc->config.dpll_hw_state.dpll_md = dpll_md; + mutex_lock(&dev_priv->dpio_lock); - if (crtc->config.has_dp_encoder) - intel_dp_set_m_n(crtc); + /* p1 and p2 divider */ + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port), + 5 << DPIO_CHV_S1_DIV_SHIFT | + bestp1 << DPIO_CHV_P1_DIV_SHIFT | + bestp2 << DPIO_CHV_P2_DIV_SHIFT | + 1 << DPIO_CHV_K_DIV_SHIFT); + + /* Feedback post-divider - m2 */ + vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2); + + /* Feedback refclk divider - n and m1 */ + vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port), + DPIO_CHV_M1_DIV_BY_2 | + 1 << DPIO_CHV_N_DIV_SHIFT); + + /* M2 fraction division */ + vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac); + + /* M2 fraction division enable */ + vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), + DPIO_CHV_FRAC_DIV_EN | + (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT)); + + /* Loop filter */ + refclk = i9xx_get_refclk(&crtc->base, 0); + loopfilter = 5 << DPIO_CHV_PROP_COEFF_SHIFT | + 2 << DPIO_CHV_GAIN_CTRL_SHIFT; + if (refclk == 100000) + intcoeff = 11; + else if (refclk == 38400) + intcoeff = 10; + else + intcoeff = 9; + loopfilter |= intcoeff << DPIO_CHV_INT_COEFF_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter); + + /* AFC Recal */ + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), + vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) | + DPIO_AFC_RECAL); mutex_unlock(&dev_priv->dpio_lock); } @@ -5111,9 +5702,6 @@ static void i9xx_update_pll(struct intel_crtc *crtc, << DPLL_MD_UDI_MULTIPLIER_SHIFT; crtc->config.dpll_hw_state.dpll_md = dpll_md; } - - if (crtc->config.has_dp_encoder) - intel_dp_set_m_n(crtc); } static void i8xx_update_pll(struct intel_crtc *crtc, @@ -5161,21 +5749,26 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder; struct drm_display_mode *adjusted_mode = &intel_crtc->config.adjusted_mode; - uint32_t vsyncshift, crtc_vtotal, crtc_vblank_end; + uint32_t crtc_vtotal, crtc_vblank_end; + int vsyncshift = 0; /* We need to be careful not to changed the adjusted mode, for otherwise * the hw state checker will get angry at the mismatch. */ crtc_vtotal = adjusted_mode->crtc_vtotal; crtc_vblank_end = adjusted_mode->crtc_vblank_end; - if (!IS_GEN2(dev) && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { + if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { /* the chip adds 2 halflines automatically */ crtc_vtotal -= 1; crtc_vblank_end -= 1; - vsyncshift = adjusted_mode->crtc_hsync_start - - adjusted_mode->crtc_htotal / 2; - } else { - vsyncshift = 0; + + if (intel_pipe_has_type(&intel_crtc->base, INTEL_OUTPUT_SDVO)) + vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2; + else + vsyncshift = adjusted_mode->crtc_hsync_start - + adjusted_mode->crtc_htotal / 2; + if (vsyncshift < 0) + vsyncshift += adjusted_mode->crtc_htotal; } if (INTEL_INFO(dev)->gen > 3) @@ -5259,25 +5852,23 @@ static void intel_get_pipe_timings(struct intel_crtc *crtc, pipe_config->requested_mode.hdisplay = pipe_config->pipe_src_w; } -static void intel_crtc_mode_from_pipe_config(struct intel_crtc *intel_crtc, - struct intel_crtc_config *pipe_config) +void intel_mode_from_pipe_config(struct drm_display_mode *mode, + struct intel_crtc_config *pipe_config) { - struct drm_crtc *crtc = &intel_crtc->base; - - crtc->mode.hdisplay = pipe_config->adjusted_mode.crtc_hdisplay; - crtc->mode.htotal = pipe_config->adjusted_mode.crtc_htotal; - crtc->mode.hsync_start = pipe_config->adjusted_mode.crtc_hsync_start; - crtc->mode.hsync_end = pipe_config->adjusted_mode.crtc_hsync_end; + mode->hdisplay = pipe_config->adjusted_mode.crtc_hdisplay; + mode->htotal = pipe_config->adjusted_mode.crtc_htotal; + mode->hsync_start = pipe_config->adjusted_mode.crtc_hsync_start; + mode->hsync_end = pipe_config->adjusted_mode.crtc_hsync_end; - crtc->mode.vdisplay = pipe_config->adjusted_mode.crtc_vdisplay; - crtc->mode.vtotal = pipe_config->adjusted_mode.crtc_vtotal; - crtc->mode.vsync_start = pipe_config->adjusted_mode.crtc_vsync_start; - crtc->mode.vsync_end = pipe_config->adjusted_mode.crtc_vsync_end; + mode->vdisplay = pipe_config->adjusted_mode.crtc_vdisplay; + mode->vtotal = pipe_config->adjusted_mode.crtc_vtotal; + mode->vsync_start = pipe_config->adjusted_mode.crtc_vsync_start; + mode->vsync_end = pipe_config->adjusted_mode.crtc_vsync_end; - crtc->mode.flags = pipe_config->adjusted_mode.flags; + mode->flags = pipe_config->adjusted_mode.flags; - crtc->mode.clock = pipe_config->adjusted_mode.crtc_clock; - crtc->mode.flags |= pipe_config->adjusted_mode.flags; + mode->clock = pipe_config->adjusted_mode.crtc_clock; + mode->flags |= pipe_config->adjusted_mode.flags; } static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) @@ -5327,10 +5918,13 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) } } - if (!IS_GEN2(dev) && - intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) - pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; - else + if (intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { + if (INTEL_INFO(dev)->gen < 4 || + intel_pipe_has_type(&intel_crtc->base, INTEL_OUTPUT_SDVO)) + pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; + else + pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT; + } else pipeconf |= PIPECONF_PROGRESSIVE; if (IS_VALLEYVIEW(dev) && intel_crtc->config.limited_color_range) @@ -5347,16 +5941,12 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; int refclk, num_connectors = 0; intel_clock_t clock, reduced_clock; - u32 dspcntr; bool ok, has_reduced_clock = false; bool is_lvds = false, is_dsi = false; struct intel_encoder *encoder; const intel_limit_t *limit; - int ret; for_each_encoder_on_crtc(dev, crtc, encoder) { switch (encoder->type) { @@ -5372,7 +5962,7 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc, } if (is_dsi) - goto skip_dpll; + return 0; if (!intel_crtc->config.clock_set) { refclk = i9xx_get_refclk(crtc, num_connectors); @@ -5417,43 +6007,17 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc, i8xx_update_pll(intel_crtc, has_reduced_clock ? &reduced_clock : NULL, num_connectors); + } else if (IS_CHERRYVIEW(dev)) { + chv_update_pll(intel_crtc); } else if (IS_VALLEYVIEW(dev)) { vlv_update_pll(intel_crtc); } else { i9xx_update_pll(intel_crtc, has_reduced_clock ? &reduced_clock : NULL, - num_connectors); - } - -skip_dpll: - /* Set up the display plane register */ - dspcntr = DISPPLANE_GAMMA_ENABLE; - - if (!IS_VALLEYVIEW(dev)) { - if (pipe == 0) - dspcntr &= ~DISPPLANE_SEL_PIPE_MASK; - else - dspcntr |= DISPPLANE_SEL_PIPE_B; + num_connectors); } - intel_set_pipe_timings(intel_crtc); - - /* pipesrc and dspsize control the size that is scaled from, - * which should always be the user's requested size. - */ - I915_WRITE(DSPSIZE(plane), - ((intel_crtc->config.pipe_src_h - 1) << 16) | - (intel_crtc->config.pipe_src_w - 1)); - I915_WRITE(DSPPOS(plane), 0); - - i9xx_set_pipeconf(intel_crtc); - - I915_WRITE(DSPCNTR(plane), dspcntr); - POSTING_READ(DSPCNTR(plane)); - - ret = intel_pipe_set_base(crtc, x, y, fb); - - return ret; + return 0; } static void i9xx_get_pfit_config(struct intel_crtc *crtc, @@ -5512,6 +6076,97 @@ static void vlv_crtc_clock_get(struct intel_crtc *crtc, pipe_config->port_clock = clock.dot / 5; } +static void i9xx_get_plane_config(struct intel_crtc *crtc, + struct intel_plane_config *plane_config) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 val, base, offset; + int pipe = crtc->pipe, plane = crtc->plane; + int fourcc, pixel_format; + int aligned_height; + + crtc->base.primary->fb = kzalloc(sizeof(struct intel_framebuffer), GFP_KERNEL); + if (!crtc->base.primary->fb) { + DRM_DEBUG_KMS("failed to alloc fb\n"); + return; + } + + val = I915_READ(DSPCNTR(plane)); + + if (INTEL_INFO(dev)->gen >= 4) + if (val & DISPPLANE_TILED) + plane_config->tiled = true; + + pixel_format = val & DISPPLANE_PIXFORMAT_MASK; + fourcc = intel_format_to_fourcc(pixel_format); + crtc->base.primary->fb->pixel_format = fourcc; + crtc->base.primary->fb->bits_per_pixel = + drm_format_plane_cpp(fourcc, 0) * 8; + + if (INTEL_INFO(dev)->gen >= 4) { + if (plane_config->tiled) + offset = I915_READ(DSPTILEOFF(plane)); + else + offset = I915_READ(DSPLINOFF(plane)); + base = I915_READ(DSPSURF(plane)) & 0xfffff000; + } else { + base = I915_READ(DSPADDR(plane)); + } + plane_config->base = base; + + val = I915_READ(PIPESRC(pipe)); + crtc->base.primary->fb->width = ((val >> 16) & 0xfff) + 1; + crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1; + + val = I915_READ(DSPSTRIDE(pipe)); + crtc->base.primary->fb->pitches[0] = val & 0xffffff80; + + aligned_height = intel_align_height(dev, crtc->base.primary->fb->height, + plane_config->tiled); + + plane_config->size = ALIGN(crtc->base.primary->fb->pitches[0] * + aligned_height, PAGE_SIZE); + + DRM_DEBUG_KMS("pipe/plane %d/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n", + pipe, plane, crtc->base.primary->fb->width, + crtc->base.primary->fb->height, + crtc->base.primary->fb->bits_per_pixel, base, + crtc->base.primary->fb->pitches[0], + plane_config->size); + +} + +static void chv_crtc_clock_get(struct intel_crtc *crtc, + struct intel_crtc_config *pipe_config) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int pipe = pipe_config->cpu_transcoder; + enum dpio_channel port = vlv_pipe_to_channel(pipe); + intel_clock_t clock; + u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2; + int refclk = 100000; + + mutex_lock(&dev_priv->dpio_lock); + cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port)); + pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port)); + pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port)); + pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port)); + mutex_unlock(&dev_priv->dpio_lock); + + clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0; + clock.m2 = ((pll_dw0 & 0xff) << 22) | (pll_dw2 & 0x3fffff); + clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf; + clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7; + clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f; + + chv_clock(refclk, &clock); + + /* clock.dot is the fast clock */ + pipe_config->port_clock = clock.dot / 5; +} + static bool i9xx_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config) { @@ -5519,6 +6174,10 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, struct drm_i915_private *dev_priv = dev->dev_private; uint32_t tmp; + if (!intel_display_power_enabled(dev_priv, + POWER_DOMAIN_PIPE(crtc->pipe))) + return false; + pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; pipe_config->shared_dpll = DPLL_ID_PRIVATE; @@ -5542,6 +6201,9 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, } } + if (IS_VALLEYVIEW(dev) && (tmp & PIPECONF_COLOR_RANGE_SELECT)) + pipe_config->limited_color_range = true; + if (INTEL_INFO(dev)->gen < 4) pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE; @@ -5577,7 +6239,9 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, DPLL_PORTB_READY_MASK); } - if (IS_VALLEYVIEW(dev)) + if (IS_CHERRYVIEW(dev)) + chv_crtc_clock_get(crtc, pipe_config); + else if (IS_VALLEYVIEW(dev)) vlv_crtc_clock_get(crtc, pipe_config); else i9xx_crtc_clock_get(crtc, pipe_config); @@ -5698,8 +6362,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev) if (intel_panel_use_ssc(dev_priv) && can_ssc) { DRM_DEBUG_KMS("Using SSC on eDP\n"); val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD; - } - else + } else val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD; } else val |= DREF_CPU_SOURCE_OUTPUT_DISABLE; @@ -6180,7 +6843,7 @@ int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp) * is 2.5%; use 5% for safety's sake. */ u32 bps = target_clock * bpp * 21 / 20; - return bps / (link_bw * 8) + 1; + return DIV_ROUND_UP(bps, link_bw * 8); } static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor) @@ -6278,10 +6941,7 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc, struct drm_framebuffer *fb) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int pipe = intel_crtc->pipe; - int plane = intel_crtc->plane; int num_connectors = 0; intel_clock_t clock, reduced_clock; u32 dpll = 0, fp = 0, fp2 = 0; @@ -6289,7 +6949,6 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc, bool is_lvds = false; struct intel_encoder *encoder; struct intel_shared_dpll *pll; - int ret; for_each_encoder_on_crtc(dev, crtc, encoder) { switch (encoder->type) { @@ -6339,36 +6998,18 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc, pll = intel_get_shared_dpll(intel_crtc); if (pll == NULL) { DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n", - pipe_name(pipe)); + pipe_name(intel_crtc->pipe)); return -EINVAL; } } else intel_put_shared_dpll(intel_crtc); - if (intel_crtc->config.has_dp_encoder) - intel_dp_set_m_n(intel_crtc); - - if (is_lvds && has_reduced_clock && i915_powersave) + if (is_lvds && has_reduced_clock && i915.powersave) intel_crtc->lowfreq_avail = true; else intel_crtc->lowfreq_avail = false; - intel_set_pipe_timings(intel_crtc); - - if (intel_crtc->config.has_pch_encoder) { - intel_cpu_transcoder_set_m_n(intel_crtc, - &intel_crtc->config.fdi_m_n); - } - - ironlake_set_pipeconf(crtc); - - /* Set up the display plane register */ - I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE); - POSTING_READ(DSPCNTR(plane)); - - ret = intel_pipe_set_base(crtc, x, y, fb); - - return ret; + return 0; } static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc, @@ -6455,6 +7096,66 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc, } } +static void ironlake_get_plane_config(struct intel_crtc *crtc, + struct intel_plane_config *plane_config) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 val, base, offset; + int pipe = crtc->pipe, plane = crtc->plane; + int fourcc, pixel_format; + int aligned_height; + + crtc->base.primary->fb = kzalloc(sizeof(struct intel_framebuffer), GFP_KERNEL); + if (!crtc->base.primary->fb) { + DRM_DEBUG_KMS("failed to alloc fb\n"); + return; + } + + val = I915_READ(DSPCNTR(plane)); + + if (INTEL_INFO(dev)->gen >= 4) + if (val & DISPPLANE_TILED) + plane_config->tiled = true; + + pixel_format = val & DISPPLANE_PIXFORMAT_MASK; + fourcc = intel_format_to_fourcc(pixel_format); + crtc->base.primary->fb->pixel_format = fourcc; + crtc->base.primary->fb->bits_per_pixel = + drm_format_plane_cpp(fourcc, 0) * 8; + + base = I915_READ(DSPSURF(plane)) & 0xfffff000; + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + offset = I915_READ(DSPOFFSET(plane)); + } else { + if (plane_config->tiled) + offset = I915_READ(DSPTILEOFF(plane)); + else + offset = I915_READ(DSPLINOFF(plane)); + } + plane_config->base = base; + + val = I915_READ(PIPESRC(pipe)); + crtc->base.primary->fb->width = ((val >> 16) & 0xfff) + 1; + crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1; + + val = I915_READ(DSPSTRIDE(pipe)); + crtc->base.primary->fb->pitches[0] = val & 0xffffff80; + + aligned_height = intel_align_height(dev, crtc->base.primary->fb->height, + plane_config->tiled); + + plane_config->size = ALIGN(crtc->base.primary->fb->pitches[0] * + aligned_height, PAGE_SIZE); + + DRM_DEBUG_KMS("pipe/plane %d/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n", + pipe, plane, crtc->base.primary->fb->width, + crtc->base.primary->fb->height, + crtc->base.primary->fb->bits_per_pixel, base, + crtc->base.primary->fb->pitches[0], + plane_config->size); +} + static bool ironlake_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config) { @@ -6486,6 +7187,9 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc, break; } + if (tmp & PIPECONF_COLOR_RANGE_SELECT) + pipe_config->limited_color_range = true; + if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) { struct intel_shared_dpll *pll; @@ -6535,10 +7239,8 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) struct drm_device *dev = dev_priv->dev; struct intel_ddi_plls *plls = &dev_priv->ddi_plls; struct intel_crtc *crtc; - unsigned long irqflags; - uint32_t val; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) + for_each_intel_crtc(dev, crtc) WARN(crtc->active, "CRTC for pipe %c enabled\n", pipe_name(crtc->pipe)); @@ -6557,14 +7259,29 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) "Utility pin enabled\n"); WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n"); - spin_lock_irqsave(&dev_priv->irq_lock, irqflags); - val = I915_READ(DEIMR); - WARN((val | DE_PCH_EVENT_IVB) != 0xffffffff, - "Unexpected DEIMR bits enabled: 0x%x\n", val); - val = I915_READ(SDEIMR); - WARN((val | SDE_HOTPLUG_MASK_CPT) != 0xffffffff, - "Unexpected SDEIMR bits enabled: 0x%x\n", val); - spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + /* + * In theory we can still leave IRQs enabled, as long as only the HPD + * interrupts remain enabled. We used to check for that, but since it's + * gen-specific and since we only disable LCPLL after we fully disable + * the interrupts, the check below should be enough. + */ + WARN(!dev_priv->pm.irqs_disabled, "IRQs enabled\n"); +} + +static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val) +{ + struct drm_device *dev = dev_priv->dev; + + if (IS_HASWELL(dev)) { + mutex_lock(&dev_priv->rps.hw_lock); + if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP, + val)) + DRM_ERROR("Failed to disable D_COMP\n"); + mutex_unlock(&dev_priv->rps.hw_lock); + } else { + I915_WRITE(D_COMP, val); + } + POSTING_READ(D_COMP); } /* @@ -6604,11 +7321,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, val = I915_READ(D_COMP); val |= D_COMP_COMP_DISABLE; - mutex_lock(&dev_priv->rps.hw_lock); - if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP, val)) - DRM_ERROR("Failed to disable D_COMP\n"); - mutex_unlock(&dev_priv->rps.hw_lock); - POSTING_READ(D_COMP); + hsw_write_dcomp(dev_priv, val); ndelay(100); if (wait_for((I915_READ(D_COMP) & D_COMP_RCOMP_IN_PROGRESS) == 0, 1)) @@ -6629,6 +7342,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) { uint32_t val; + unsigned long irqflags; val = I915_READ(LCPLL_CTL); @@ -6636,9 +7350,22 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK) return; - /* Make sure we're not on PC8 state before disabling PC8, otherwise - * we'll hang the machine! */ - gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); + /* + * Make sure we're not on PC8 state before disabling PC8, otherwise + * we'll hang the machine. To prevent PC8 state, just enable force_wake. + * + * The other problem is that hsw_restore_lcpll() is called as part of + * the runtime PM resume sequence, so we can't just call + * gen6_gt_force_wake_get() because that function calls + * intel_runtime_pm_get(), and we can't change the runtime PM refcount + * while we are on the resume sequence. So to solve this problem we have + * to call special forcewake code that doesn't touch runtime PM and + * doesn't enable the forcewake delayed work. + */ + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + if (dev_priv->uncore.forcewake_count++ == 0) + dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); if (val & LCPLL_POWER_DOWN_ALLOW) { val &= ~LCPLL_POWER_DOWN_ALLOW; @@ -6649,11 +7376,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) val = I915_READ(D_COMP); val |= D_COMP_COMP_FORCE; val &= ~D_COMP_COMP_DISABLE; - mutex_lock(&dev_priv->rps.hw_lock); - if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP, val)) - DRM_ERROR("Failed to enable D_COMP\n"); - mutex_unlock(&dev_priv->rps.hw_lock); - POSTING_READ(D_COMP); + hsw_write_dcomp(dev_priv, val); val = I915_READ(LCPLL_CTL); val &= ~LCPLL_PLL_DISABLE; @@ -6672,26 +7395,43 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) DRM_ERROR("Switching back to LCPLL failed\n"); } - gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); + /* See the big comment above. */ + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + if (--dev_priv->uncore.forcewake_count == 0) + dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } -void hsw_enable_pc8_work(struct work_struct *__work) +/* + * Package states C8 and deeper are really deep PC states that can only be + * reached when all the devices on the system allow it, so even if the graphics + * device allows PC8+, it doesn't mean the system will actually get to these + * states. Our driver only allows PC8+ when going into runtime PM. + * + * The requirements for PC8+ are that all the outputs are disabled, the power + * well is disabled and most interrupts are disabled, and these are also + * requirements for runtime PM. When these conditions are met, we manually do + * the other conditions: disable the interrupts, clocks and switch LCPLL refclk + * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard + * hang the machine. + * + * When we really reach PC8 or deeper states (not just when we allow it) we lose + * the state of some registers, so when we come back from PC8+ we need to + * restore this state. We don't get into PC8+ if we're not in RC6, so we don't + * need to take care of the registers kept by RC6. Notice that this happens even + * if we don't put the device in PCI D3 state (which is what currently happens + * because of the runtime PM support). + * + * For more, read "Display Sequences for Package C8" on the hardware + * documentation. + */ +void hsw_enable_pc8(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = - container_of(to_delayed_work(__work), struct drm_i915_private, - pc8.enable_work); struct drm_device *dev = dev_priv->dev; uint32_t val; - WARN_ON(!HAS_PC8(dev)); - - if (dev_priv->pc8.enabled) - return; - DRM_DEBUG_KMS("Enabling package C8+\n"); - dev_priv->pc8.enabled = true; - if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { val = I915_READ(SOUTH_DSPCLK_GATE_D); val &= ~PCH_LP_PARTITION_LEVEL_DISABLE; @@ -6699,51 +7439,17 @@ void hsw_enable_pc8_work(struct work_struct *__work) } lpt_disable_clkout_dp(dev); - hsw_pc8_disable_interrupts(dev); hsw_disable_lcpll(dev_priv, true, true); - - intel_runtime_pm_put(dev_priv); } -static void __hsw_enable_package_c8(struct drm_i915_private *dev_priv) -{ - WARN_ON(!mutex_is_locked(&dev_priv->pc8.lock)); - WARN(dev_priv->pc8.disable_count < 1, - "pc8.disable_count: %d\n", dev_priv->pc8.disable_count); - - dev_priv->pc8.disable_count--; - if (dev_priv->pc8.disable_count != 0) - return; - - schedule_delayed_work(&dev_priv->pc8.enable_work, - msecs_to_jiffies(i915_pc8_timeout)); -} - -static void __hsw_disable_package_c8(struct drm_i915_private *dev_priv) +void hsw_disable_pc8(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; uint32_t val; - WARN_ON(!mutex_is_locked(&dev_priv->pc8.lock)); - WARN(dev_priv->pc8.disable_count < 0, - "pc8.disable_count: %d\n", dev_priv->pc8.disable_count); - - dev_priv->pc8.disable_count++; - if (dev_priv->pc8.disable_count != 1) - return; - - WARN_ON(!HAS_PC8(dev)); - - cancel_delayed_work_sync(&dev_priv->pc8.enable_work); - if (!dev_priv->pc8.enabled) - return; - DRM_DEBUG_KMS("Disabling package C8+\n"); - intel_runtime_pm_get(dev_priv); - hsw_restore_lcpll(dev_priv); - hsw_pc8_restore_interrupts(dev); lpt_init_pch_refclk(dev); if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) { @@ -6753,228 +7459,31 @@ static void __hsw_disable_package_c8(struct drm_i915_private *dev_priv) } intel_prepare_ddi(dev); - i915_gem_init_swizzling(dev); - mutex_lock(&dev_priv->rps.hw_lock); - gen6_update_ring_freq(dev); - mutex_unlock(&dev_priv->rps.hw_lock); - dev_priv->pc8.enabled = false; } -void hsw_enable_package_c8(struct drm_i915_private *dev_priv) +static void snb_modeset_global_resources(struct drm_device *dev) { - if (!HAS_PC8(dev_priv->dev)) - return; - - mutex_lock(&dev_priv->pc8.lock); - __hsw_enable_package_c8(dev_priv); - mutex_unlock(&dev_priv->pc8.lock); -} - -void hsw_disable_package_c8(struct drm_i915_private *dev_priv) -{ - if (!HAS_PC8(dev_priv->dev)) - return; - - mutex_lock(&dev_priv->pc8.lock); - __hsw_disable_package_c8(dev_priv); - mutex_unlock(&dev_priv->pc8.lock); -} - -static bool hsw_can_enable_package_c8(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - struct intel_crtc *crtc; - uint32_t val; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) - if (crtc->base.enabled) - return false; - - /* This case is still possible since we have the i915.disable_power_well - * parameter and also the KVMr or something else might be requesting the - * power well. */ - val = I915_READ(HSW_PWR_WELL_DRIVER); - if (val != 0) { - DRM_DEBUG_KMS("Not enabling PC8: power well on\n"); - return false; - } - - return true; -} - -/* Since we're called from modeset_global_resources there's no way to - * symmetrically increase and decrease the refcount, so we use - * dev_priv->pc8.requirements_met to track whether we already have the refcount - * or not. - */ -static void hsw_update_package_c8(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - bool allow; - - if (!HAS_PC8(dev_priv->dev)) - return; - - if (!i915_enable_pc8) - return; - - mutex_lock(&dev_priv->pc8.lock); - - allow = hsw_can_enable_package_c8(dev_priv); - - if (allow == dev_priv->pc8.requirements_met) - goto done; - - dev_priv->pc8.requirements_met = allow; - - if (allow) - __hsw_enable_package_c8(dev_priv); - else - __hsw_disable_package_c8(dev_priv); - -done: - mutex_unlock(&dev_priv->pc8.lock); -} - -static void hsw_package_c8_gpu_idle(struct drm_i915_private *dev_priv) -{ - if (!HAS_PC8(dev_priv->dev)) - return; - - mutex_lock(&dev_priv->pc8.lock); - if (!dev_priv->pc8.gpu_idle) { - dev_priv->pc8.gpu_idle = true; - __hsw_enable_package_c8(dev_priv); - } - mutex_unlock(&dev_priv->pc8.lock); -} - -static void hsw_package_c8_gpu_busy(struct drm_i915_private *dev_priv) -{ - if (!HAS_PC8(dev_priv->dev)) - return; - - mutex_lock(&dev_priv->pc8.lock); - if (dev_priv->pc8.gpu_idle) { - dev_priv->pc8.gpu_idle = false; - __hsw_disable_package_c8(dev_priv); - } - mutex_unlock(&dev_priv->pc8.lock); -} - -#define for_each_power_domain(domain, mask) \ - for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \ - if ((1 << (domain)) & (mask)) - -static unsigned long get_pipe_power_domains(struct drm_device *dev, - enum pipe pipe, bool pfit_enabled) -{ - unsigned long mask; - enum transcoder transcoder; - - transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe); - - mask = BIT(POWER_DOMAIN_PIPE(pipe)); - mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder)); - if (pfit_enabled) - mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe)); - - return mask; -} - -void intel_display_set_init_power(struct drm_device *dev, bool enable) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - if (dev_priv->power_domains.init_power_on == enable) - return; - - if (enable) - intel_display_power_get(dev, POWER_DOMAIN_INIT); - else - intel_display_power_put(dev, POWER_DOMAIN_INIT); - - dev_priv->power_domains.init_power_on = enable; -} - -static void modeset_update_power_wells(struct drm_device *dev) -{ - unsigned long pipe_domains[I915_MAX_PIPES] = { 0, }; - struct intel_crtc *crtc; - - /* - * First get all needed power domains, then put all unneeded, to avoid - * any unnecessary toggling of the power wells. - */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) { - enum intel_display_power_domain domain; - - if (!crtc->base.enabled) - continue; - - pipe_domains[crtc->pipe] = get_pipe_power_domains(dev, - crtc->pipe, - crtc->config.pch_pfit.enabled); - - for_each_power_domain(domain, pipe_domains[crtc->pipe]) - intel_display_power_get(dev, domain); - } - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) { - enum intel_display_power_domain domain; - - for_each_power_domain(domain, crtc->enabled_power_domains) - intel_display_power_put(dev, domain); - - crtc->enabled_power_domains = pipe_domains[crtc->pipe]; - } - - intel_display_set_init_power(dev, false); + modeset_update_crtc_power_domains(dev); } static void haswell_modeset_global_resources(struct drm_device *dev) { - modeset_update_power_wells(dev); - hsw_update_package_c8(dev); + modeset_update_crtc_power_domains(dev); } static int haswell_crtc_mode_set(struct drm_crtc *crtc, int x, int y, struct drm_framebuffer *fb) { - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane; - int ret; if (!intel_ddi_pll_select(intel_crtc)) return -EINVAL; intel_ddi_pll_enable(intel_crtc); - if (intel_crtc->config.has_dp_encoder) - intel_dp_set_m_n(intel_crtc); - intel_crtc->lowfreq_avail = false; - intel_set_pipe_timings(intel_crtc); - - if (intel_crtc->config.has_pch_encoder) { - intel_cpu_transcoder_set_m_n(intel_crtc, - &intel_crtc->config.fdi_m_n); - } - - haswell_set_pipeconf(crtc); - - intel_set_pipe_csc(crtc); - - /* Set up the display plane register */ - I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE | DISPPLANE_PIPE_CSC_ENABLE); - POSTING_READ(DSPCNTR(plane)); - - ret = intel_pipe_set_base(crtc, x, y, fb); - - return ret; + return 0; } static bool haswell_get_pipe_config(struct intel_crtc *crtc, @@ -6985,6 +7494,10 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, enum intel_display_power_domain pfit_domain; uint32_t tmp; + if (!intel_display_power_enabled(dev_priv, + POWER_DOMAIN_PIPE(crtc->pipe))) + return false; + pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; pipe_config->shared_dpll = DPLL_ID_PRIVATE; @@ -7010,7 +7523,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, pipe_config->cpu_transcoder = TRANSCODER_EDP; } - if (!intel_display_power_enabled(dev, + if (!intel_display_power_enabled(dev_priv, POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder))) return false; @@ -7038,7 +7551,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, intel_get_pipe_timings(crtc, pipe_config); pfit_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe); - if (intel_display_power_enabled(dev, pfit_domain)) + if (intel_display_power_enabled(dev_priv, pfit_domain)) ironlake_get_pfit_config(crtc, pipe_config); if (IS_HASWELL(dev)) @@ -7050,38 +7563,6 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, return true; } -static int intel_crtc_mode_set(struct drm_crtc *crtc, - int x, int y, - struct drm_framebuffer *fb) -{ - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_encoder *encoder; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_display_mode *mode = &intel_crtc->config.requested_mode; - int pipe = intel_crtc->pipe; - int ret; - - drm_vblank_pre_modeset(dev, pipe); - - ret = dev_priv->display.crtc_mode_set(crtc, x, y, fb); - - drm_vblank_post_modeset(dev, pipe); - - if (ret != 0) - return ret; - - for_each_encoder_on_crtc(dev, crtc, encoder) { - DRM_DEBUG_KMS("[ENCODER:%d:%s] set [MODE:%d:%s]\n", - encoder->base.base.id, - drm_get_encoder_name(&encoder->base), - mode->base.id, mode->name); - encoder->mode_set(encoder); - } - - return 0; -} - static struct { int clock; u32 config; @@ -7196,8 +7677,6 @@ static void haswell_write_eld(struct drm_connector *connector, { struct drm_i915_private *dev_priv = connector->dev->dev_private; uint8_t *eld = connector->eld; - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t eldv; uint32_t i; int len; @@ -7209,17 +7688,14 @@ static void haswell_write_eld(struct drm_connector *connector, int aud_config = HSW_AUD_CFG(pipe); int aud_cntrl_st2 = HSW_AUD_PIN_ELD_CP_VLD; - - DRM_DEBUG_DRIVER("HDMI: Haswell Audio initialize....\n"); - /* Audio output enable */ DRM_DEBUG_DRIVER("HDMI audio: enable codec\n"); tmp = I915_READ(aud_cntrl_st2); tmp |= (AUDIO_OUTPUT_ENABLE_A << (pipe * 4)); I915_WRITE(aud_cntrl_st2, tmp); + POSTING_READ(aud_cntrl_st2); - /* Wait for 1 vertical blank */ - intel_wait_for_vblank(dev, pipe); + assert_pipe_disabled(dev_priv, to_intel_crtc(crtc)->pipe); /* Set ELD valid state */ tmp = I915_READ(aud_cntrl_st2); @@ -7239,7 +7715,6 @@ static void haswell_write_eld(struct drm_connector *connector, DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(pipe)); eldv = AUDIO_ELD_VALID_A << (pipe * 4); - intel_crtc->eld_vld = true; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) { DRM_DEBUG_DRIVER("ELD: DisplayPort detected\n"); @@ -7386,9 +7861,9 @@ void intel_write_eld(struct drm_encoder *encoder, DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", connector->base.id, - drm_get_connector_name(connector), + connector->name, connector->encoder->base.id, - drm_get_encoder_name(connector->encoder)); + connector->encoder->name); connector->eld[6] = drm_av_sync_delay(connector, mode) / 2; @@ -7401,29 +7876,33 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base) struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - bool visible = base != 0; - u32 cntl; - - if (intel_crtc->cursor_visible == visible) - return; + uint32_t cntl; - cntl = I915_READ(_CURACNTR); - if (visible) { + if (base != intel_crtc->cursor_base) { /* On these chipsets we can only modify the base whilst * the cursor is disabled. */ + if (intel_crtc->cursor_cntl) { + I915_WRITE(_CURACNTR, 0); + POSTING_READ(_CURACNTR); + intel_crtc->cursor_cntl = 0; + } + I915_WRITE(_CURABASE, base); + POSTING_READ(_CURABASE); + } - cntl &= ~(CURSOR_FORMAT_MASK); - /* XXX width must be 64, stride 256 => 0x00 << 28 */ - cntl |= CURSOR_ENABLE | + /* XXX width must be 64, stride 256 => 0x00 << 28 */ + cntl = 0; + if (base) + cntl = (CURSOR_ENABLE | CURSOR_GAMMA_ENABLE | - CURSOR_FORMAT_ARGB; - } else - cntl &= ~(CURSOR_ENABLE | CURSOR_GAMMA_ENABLE); - I915_WRITE(_CURACNTR, cntl); - - intel_crtc->cursor_visible = visible; + CURSOR_FORMAT_ARGB); + if (intel_crtc->cursor_cntl != cntl) { + I915_WRITE(_CURACNTR, cntl); + POSTING_READ(_CURACNTR); + intel_crtc->cursor_cntl = cntl; + } } static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base) @@ -7432,24 +7911,34 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - bool visible = base != 0; - - if (intel_crtc->cursor_visible != visible) { - uint32_t cntl = I915_READ(CURCNTR(pipe)); - if (base) { - cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT); - cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE; - cntl |= pipe << 28; /* Connect to correct pipe */ - } else { - cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE); - cntl |= CURSOR_MODE_DISABLE; + uint32_t cntl; + + cntl = 0; + if (base) { + cntl = MCURSOR_GAMMA_ENABLE; + switch (intel_crtc->cursor_width) { + case 64: + cntl |= CURSOR_MODE_64_ARGB_AX; + break; + case 128: + cntl |= CURSOR_MODE_128_ARGB_AX; + break; + case 256: + cntl |= CURSOR_MODE_256_ARGB_AX; + break; + default: + WARN_ON(1); + return; } + cntl |= pipe << 28; /* Connect to correct pipe */ + } + if (intel_crtc->cursor_cntl != cntl) { I915_WRITE(CURCNTR(pipe), cntl); - - intel_crtc->cursor_visible = visible; + POSTING_READ(CURCNTR(pipe)); + intel_crtc->cursor_cntl = cntl; } + /* and commit changes on next vblank */ - POSTING_READ(CURCNTR(pipe)); I915_WRITE(CURBASE(pipe), base); POSTING_READ(CURBASE(pipe)); } @@ -7460,29 +7949,38 @@ static void ivb_update_cursor(struct drm_crtc *crtc, u32 base) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - bool visible = base != 0; - - if (intel_crtc->cursor_visible != visible) { - uint32_t cntl = I915_READ(CURCNTR_IVB(pipe)); - if (base) { - cntl &= ~CURSOR_MODE; - cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE; - } else { - cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE); - cntl |= CURSOR_MODE_DISABLE; - } - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { - cntl |= CURSOR_PIPE_CSC_ENABLE; - cntl &= ~CURSOR_TRICKLE_FEED_DISABLE; + uint32_t cntl; + + cntl = 0; + if (base) { + cntl = MCURSOR_GAMMA_ENABLE; + switch (intel_crtc->cursor_width) { + case 64: + cntl |= CURSOR_MODE_64_ARGB_AX; + break; + case 128: + cntl |= CURSOR_MODE_128_ARGB_AX; + break; + case 256: + cntl |= CURSOR_MODE_256_ARGB_AX; + break; + default: + WARN_ON(1); + return; } - I915_WRITE(CURCNTR_IVB(pipe), cntl); + } + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + cntl |= CURSOR_PIPE_CSC_ENABLE; - intel_crtc->cursor_visible = visible; + if (intel_crtc->cursor_cntl != cntl) { + I915_WRITE(CURCNTR(pipe), cntl); + POSTING_READ(CURCNTR(pipe)); + intel_crtc->cursor_cntl = cntl; } + /* and commit changes on next vblank */ - POSTING_READ(CURCNTR_IVB(pipe)); - I915_WRITE(CURBASE_IVB(pipe), base); - POSTING_READ(CURBASE_IVB(pipe)); + I915_WRITE(CURBASE(pipe), base); + POSTING_READ(CURBASE(pipe)); } /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */ @@ -7496,7 +7994,6 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, int x = intel_crtc->cursor_x; int y = intel_crtc->cursor_y; u32 base = 0, pos = 0; - bool visible; if (on) base = intel_crtc->cursor_addr; @@ -7525,20 +8022,18 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, } pos |= y << CURSOR_Y_SHIFT; - visible = base != 0; - if (!visible && !intel_crtc->cursor_visible) + if (base == 0 && intel_crtc->cursor_base == 0) return; - if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev)) { - I915_WRITE(CURPOS_IVB(pipe), pos); + I915_WRITE(CURPOS(pipe), pos); + + if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev)) ivb_update_cursor(crtc, base); - } else { - I915_WRITE(CURPOS(pipe), pos); - if (IS_845G(dev) || IS_I865G(dev)) - i845_update_cursor(crtc, base); - else - i9xx_update_cursor(crtc, base); - } + else if (IS_845G(dev) || IS_I865G(dev)) + i845_update_cursor(crtc, base); + else + i9xx_update_cursor(crtc, base); + intel_crtc->cursor_base = base; } static int intel_crtc_cursor_set(struct drm_crtc *crtc, @@ -7550,6 +8045,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_i915_gem_object *obj; + unsigned old_width; uint32_t addr; int ret; @@ -7562,9 +8058,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, goto finish; } - /* Currently we only support 64x64 cursors */ - if (width != 64 || height != 64) { - DRM_ERROR("we currently only support 64x64 cursors\n"); + /* Check for which cursor types we support */ + if (!((width == 64 && height == 64) || + (width == 128 && height == 128 && !IS_GEN2(dev)) || + (width == 256 && height == 256 && !IS_GEN2(dev)))) { + DRM_DEBUG("Cursor dimension not supported\n"); return -EINVAL; } @@ -7573,18 +8071,18 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, return -ENOENT; if (obj->base.size < width * height * 4) { - DRM_ERROR("buffer is to small\n"); + DRM_DEBUG_KMS("buffer is to small\n"); ret = -ENOMEM; goto fail; } /* we only need to pin inside GTT if cursor is non-phy */ mutex_lock(&dev->struct_mutex); - if (!dev_priv->info->cursor_needs_physical) { + if (!INTEL_INFO(dev)->cursor_needs_physical) { unsigned alignment; if (obj->tiling_mode) { - DRM_ERROR("cursor cannot be tiled\n"); + DRM_DEBUG_KMS("cursor cannot be tiled\n"); ret = -EINVAL; goto fail_locked; } @@ -7600,27 +8098,25 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, ret = i915_gem_object_pin_to_display_plane(obj, alignment, NULL); if (ret) { - DRM_ERROR("failed to move cursor bo into the GTT\n"); + DRM_DEBUG_KMS("failed to move cursor bo into the GTT\n"); goto fail_locked; } ret = i915_gem_object_put_fence(obj); if (ret) { - DRM_ERROR("failed to release fence for cursor"); + DRM_DEBUG_KMS("failed to release fence for cursor"); goto fail_unpin; } addr = i915_gem_obj_ggtt_offset(obj); } else { int align = IS_I830(dev) ? 16 * 1024 : 256; - ret = i915_gem_attach_phys_object(dev, obj, - (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1, - align); + ret = i915_gem_object_attach_phys(obj, align); if (ret) { - DRM_ERROR("failed to attach phys object\n"); + DRM_DEBUG_KMS("failed to attach phys object\n"); goto fail_locked; } - addr = obj->phys_obj->handle->busaddr; + addr = obj->phys_handle->busaddr; } if (IS_GEN2(dev)) @@ -7628,23 +8124,25 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, finish: if (intel_crtc->cursor_bo) { - if (dev_priv->info->cursor_needs_physical) { - if (intel_crtc->cursor_bo != obj) - i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo); - } else + if (!INTEL_INFO(dev)->cursor_needs_physical) i915_gem_object_unpin_from_display_plane(intel_crtc->cursor_bo); drm_gem_object_unreference(&intel_crtc->cursor_bo->base); } mutex_unlock(&dev->struct_mutex); + old_width = intel_crtc->cursor_width; + intel_crtc->cursor_addr = addr; intel_crtc->cursor_bo = obj; intel_crtc->cursor_width = width; intel_crtc->cursor_height = height; - if (intel_crtc->active) + if (intel_crtc->active) { + if (old_width != width) + intel_update_watermarks(crtc); intel_crtc_update_cursor(crtc, intel_crtc->cursor_bo != NULL); + } return 0; fail_unpin: @@ -7690,10 +8188,10 @@ static struct drm_display_mode load_detect_mode = { 704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC), }; -static struct drm_framebuffer * -intel_framebuffer_create(struct drm_device *dev, - struct drm_mode_fb_cmd2 *mode_cmd, - struct drm_i915_gem_object *obj) +struct drm_framebuffer * +__intel_framebuffer_create(struct drm_device *dev, + struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_i915_gem_object *obj) { struct intel_framebuffer *intel_fb; int ret; @@ -7704,12 +8202,7 @@ intel_framebuffer_create(struct drm_device *dev, return ERR_PTR(-ENOMEM); } - ret = i915_mutex_lock_interruptible(dev); - if (ret) - goto err; - ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj); - mutex_unlock(&dev->struct_mutex); if (ret) goto err; @@ -7721,6 +8214,23 @@ err: return ERR_PTR(ret); } +static struct drm_framebuffer * +intel_framebuffer_create(struct drm_device *dev, + struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_i915_gem_object *obj) +{ + struct drm_framebuffer *fb; + int ret; + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ERR_PTR(ret); + fb = __intel_framebuffer_create(dev, mode_cmd, obj); + mutex_unlock(&dev->struct_mutex); + + return fb; +} + static u32 intel_framebuffer_pitch_for_width(int width, int bpp) { @@ -7766,14 +8276,16 @@ mode_fits_in_fbdev(struct drm_device *dev, struct drm_i915_gem_object *obj; struct drm_framebuffer *fb; - if (dev_priv->fbdev == NULL) + if (!dev_priv->fbdev) return NULL; - obj = dev_priv->fbdev->ifb.obj; - if (obj == NULL) + if (!dev_priv->fbdev->fb) return NULL; - fb = &dev_priv->fbdev->ifb.base; + obj = dev_priv->fbdev->fb->obj; + BUG_ON(!obj); + + fb = &dev_priv->fbdev->fb->base; if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay, fb->bits_per_pixel)) return NULL; @@ -7789,7 +8301,8 @@ mode_fits_in_fbdev(struct drm_device *dev, bool intel_get_load_detect_pipe(struct drm_connector *connector, struct drm_display_mode *mode, - struct intel_load_detect_pipe *old) + struct intel_load_detect_pipe *old, + struct drm_modeset_acquire_ctx *ctx) { struct intel_crtc *intel_crtc; struct intel_encoder *intel_encoder = @@ -7799,11 +8312,19 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, struct drm_crtc *crtc = NULL; struct drm_device *dev = encoder->dev; struct drm_framebuffer *fb; - int i = -1; + struct drm_mode_config *config = &dev->mode_config; + int ret, i = -1; DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector), - encoder->base.id, drm_get_encoder_name(encoder)); + connector->base.id, connector->name, + encoder->base.id, encoder->name); + + drm_modeset_acquire_init(ctx, 0); + +retry: + ret = drm_modeset_lock(&config->connection_mutex, ctx); + if (ret) + goto fail_unlock; /* * Algorithm gets a little messy: @@ -7819,7 +8340,9 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, if (encoder->crtc) { crtc = encoder->crtc; - mutex_lock(&crtc->mutex); + ret = drm_modeset_lock(&crtc->mutex, ctx); + if (ret) + goto fail_unlock; old->dpms_mode = connector->dpms; old->load_detect_temp = false; @@ -7832,7 +8355,7 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, } /* Find an unused one (if possible) */ - list_for_each_entry(possible_crtc, &dev->mode_config.crtc_list, head) { + for_each_crtc(dev, possible_crtc) { i++; if (!(encoder->possible_crtcs & (1 << i))) continue; @@ -7847,14 +8370,18 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, */ if (!crtc) { DRM_DEBUG_KMS("no pipe available for load-detect\n"); - return false; + goto fail_unlock; } - mutex_lock(&crtc->mutex); + ret = drm_modeset_lock(&crtc->mutex, ctx); + if (ret) + goto fail_unlock; intel_encoder->new_crtc = to_intel_crtc(crtc); to_intel_connector(connector)->new_encoder = intel_encoder; intel_crtc = to_intel_crtc(crtc); + intel_crtc->new_enabled = true; + intel_crtc->new_config = &intel_crtc->config; old->dpms_mode = connector->dpms; old->load_detect_temp = true; old->release_fb = NULL; @@ -7878,38 +8405,57 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n"); if (IS_ERR(fb)) { DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n"); - mutex_unlock(&crtc->mutex); - return false; + goto fail; } if (intel_set_mode(crtc, mode, 0, 0, fb)) { DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n"); if (old->release_fb) old->release_fb->funcs->destroy(old->release_fb); - mutex_unlock(&crtc->mutex); - return false; + goto fail; } /* let the connector get through one full cycle before testing */ intel_wait_for_vblank(dev, intel_crtc->pipe); return true; + + fail: + intel_crtc->new_enabled = crtc->enabled; + if (intel_crtc->new_enabled) + intel_crtc->new_config = &intel_crtc->config; + else + intel_crtc->new_config = NULL; +fail_unlock: + if (ret == -EDEADLK) { + drm_modeset_backoff(ctx); + goto retry; + } + + drm_modeset_drop_locks(ctx); + drm_modeset_acquire_fini(ctx); + + return false; } void intel_release_load_detect_pipe(struct drm_connector *connector, - struct intel_load_detect_pipe *old) + struct intel_load_detect_pipe *old, + struct drm_modeset_acquire_ctx *ctx) { struct intel_encoder *intel_encoder = intel_attached_encoder(connector); struct drm_encoder *encoder = &intel_encoder->base; struct drm_crtc *crtc = encoder->crtc; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector), - encoder->base.id, drm_get_encoder_name(encoder)); + connector->base.id, connector->name, + encoder->base.id, encoder->name); if (old->load_detect_temp) { to_intel_connector(connector)->new_encoder = NULL; intel_encoder->new_crtc = NULL; + intel_crtc->new_enabled = false; + intel_crtc->new_config = NULL; intel_set_mode(crtc, NULL, 0, 0, NULL); if (old->release_fb) { @@ -7917,7 +8463,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, drm_framebuffer_unreference(old->release_fb); } - mutex_unlock(&crtc->mutex); + goto unlock; return; } @@ -7925,7 +8471,9 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, if (old->dpms_mode != DRM_MODE_DPMS_ON) connector->funcs->dpms(connector, old->dpms_mode); - mutex_unlock(&crtc->mutex); +unlock: + drm_modeset_drop_locks(ctx); + drm_modeset_acquire_fini(ctx); } static int i9xx_pll_refclk(struct drm_device *dev, @@ -8122,7 +8670,7 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev, static void intel_increase_pllclock(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; int dpll_reg = DPLL(pipe); @@ -8153,7 +8701,7 @@ static void intel_increase_pllclock(struct drm_crtc *crtc) static void intel_decrease_pllclock(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); if (HAS_PCH_SPLIT(dev)) @@ -8190,8 +8738,12 @@ void intel_mark_busy(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - hsw_package_c8_gpu_busy(dev_priv); + if (dev_priv->mm.busy) + return; + + intel_runtime_pm_get(dev_priv); i915_update_gfx_val(dev_priv); + dev_priv->mm.busy = true; } void intel_mark_idle(struct drm_device *dev) @@ -8199,36 +8751,42 @@ void intel_mark_idle(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc; - hsw_package_c8_gpu_idle(dev_priv); - - if (!i915_powersave) + if (!dev_priv->mm.busy) return; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - if (!crtc->fb) + dev_priv->mm.busy = false; + + if (!i915.powersave) + goto out; + + for_each_crtc(dev, crtc) { + if (!crtc->primary->fb) continue; intel_decrease_pllclock(crtc); } - if (dev_priv->info->gen >= 6) + if (INTEL_INFO(dev)->gen >= 6) gen6_rps_idle(dev->dev_private); + +out: + intel_runtime_pm_put(dev_priv); } void intel_mark_fb_busy(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *ring) + struct intel_engine_cs *ring) { struct drm_device *dev = obj->base.dev; struct drm_crtc *crtc; - if (!i915_powersave) + if (!i915.powersave) return; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - if (!crtc->fb) + for_each_crtc(dev, crtc) { + if (!crtc->primary->fb) continue; - if (to_intel_framebuffer(crtc->fb)->obj != obj) + if (to_intel_framebuffer(crtc->primary->fb)->obj != obj) continue; intel_increase_pllclock(crtc); @@ -8284,7 +8842,7 @@ static void intel_unpin_work_fn(struct work_struct *__work) static void do_intel_finish_page_flip(struct drm_device *dev, struct drm_crtc *crtc) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; unsigned long flags; @@ -8312,7 +8870,7 @@ static void do_intel_finish_page_flip(struct drm_device *dev, if (work->event) drm_send_vblank_event(dev, intel_crtc->pipe, work->event); - drm_vblank_put(dev, intel_crtc->pipe); + drm_crtc_vblank_put(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); @@ -8325,7 +8883,7 @@ static void do_intel_finish_page_flip(struct drm_device *dev, void intel_finish_page_flip(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; do_intel_finish_page_flip(dev, crtc); @@ -8333,15 +8891,57 @@ void intel_finish_page_flip(struct drm_device *dev, int pipe) void intel_finish_page_flip_plane(struct drm_device *dev, int plane) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane]; do_intel_finish_page_flip(dev, crtc); } +/* Is 'a' after or equal to 'b'? */ +static bool g4x_flip_count_after_eq(u32 a, u32 b) +{ + return !((a - b) & 0x80000000); +} + +static bool page_flip_finished(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + /* + * The relevant registers doen't exist on pre-ctg. + * As the flip done interrupt doesn't trigger for mmio + * flips on gmch platforms, a flip count check isn't + * really needed there. But since ctg has the registers, + * include it in the check anyway. + */ + if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev)) + return true; + + /* + * A DSPSURFLIVE check isn't enough in case the mmio and CS flips + * used the same base address. In that case the mmio flip might + * have completed, but the CS hasn't even executed the flip yet. + * + * A flip count check isn't enough as the CS might have updated + * the base address just after start of vblank, but before we + * managed to process the interrupt. This means we'd complete the + * CS flip too soon. + * + * Combining both checks should get us a good enough result. It may + * still happen that the CS flip has been executed, but has not + * yet actually completed. But in case the base address is the same + * anyway, we don't really care. + */ + return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) == + crtc->unpin_work->gtt_offset && + g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_GM45(crtc->pipe)), + crtc->unpin_work->flip_count); +} + void intel_prepare_page_flip(struct drm_device *dev, int plane) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]); unsigned long flags; @@ -8351,12 +8951,12 @@ void intel_prepare_page_flip(struct drm_device *dev, int plane) * is also accompanied by a spurious intel_prepare_page_flip(). */ spin_lock_irqsave(&dev->event_lock, flags); - if (intel_crtc->unpin_work) + if (intel_crtc->unpin_work && page_flip_finished(intel_crtc)) atomic_inc_not_zero(&intel_crtc->unpin_work->pending); spin_unlock_irqrestore(&dev->event_lock, flags); } -inline static void intel_mark_page_flip_active(struct intel_crtc *intel_crtc) +static inline void intel_mark_page_flip_active(struct intel_crtc *intel_crtc) { /* Ensure that the work item is consistent when activating it ... */ smp_wmb(); @@ -8369,21 +8969,16 @@ static int intel_gen2_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 flip_mask; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; int ret; - ret = intel_pin_and_fence_fb_obj(dev, obj, ring); - if (ret) - goto err; - ret = intel_ring_begin(ring, 6); if (ret) - goto err_unpin; + return ret; /* Can't queue multiple flips, so wait for the previous * one to finish before executing the next. @@ -8397,38 +8992,28 @@ static int intel_gen2_queue_flip(struct drm_device *dev, intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset); + intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); intel_ring_emit(ring, 0); /* aux display base address, unused */ intel_mark_page_flip_active(intel_crtc); __intel_ring_advance(ring); return 0; - -err_unpin: - intel_unpin_fb_obj(obj); -err: - return ret; } static int intel_gen3_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); u32 flip_mask; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; int ret; - ret = intel_pin_and_fence_fb_obj(dev, obj, ring); - if (ret) - goto err; - ret = intel_ring_begin(ring, 6); if (ret) - goto err_unpin; + return ret; if (intel_crtc->plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; @@ -8439,38 +9024,29 @@ static int intel_gen3_queue_flip(struct drm_device *dev, intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset); + intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); intel_ring_emit(ring, MI_NOOP); intel_mark_page_flip_active(intel_crtc); __intel_ring_advance(ring); return 0; - -err_unpin: - intel_unpin_fb_obj(obj); -err: - return ret; } static int intel_gen4_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pf, pipesrc; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; int ret; - ret = intel_pin_and_fence_fb_obj(dev, obj, ring); - if (ret) - goto err; - ret = intel_ring_begin(ring, 4); if (ret) - goto err_unpin; + return ret; /* i965+ uses the linear or tiled offsets from the * Display Registers (which do not change across a page-flip) @@ -8479,8 +9055,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev, intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0]); - intel_ring_emit(ring, - (i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset) | + intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset | obj->tiling_mode); /* XXX Enabling the panel-fitter across page-flip is so far @@ -8494,37 +9069,28 @@ static int intel_gen4_queue_flip(struct drm_device *dev, intel_mark_page_flip_active(intel_crtc); __intel_ring_advance(ring); return 0; - -err_unpin: - intel_unpin_fb_obj(obj); -err: - return ret; } static int intel_gen6_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; uint32_t pf, pipesrc; int ret; - ret = intel_pin_and_fence_fb_obj(dev, obj, ring); - if (ret) - goto err; - ret = intel_ring_begin(ring, 4); if (ret) - goto err_unpin; + return ret; intel_ring_emit(ring, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset); + intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); /* Contrary to the suggestions in the documentation, * "Enable Panel Fitter" does not seem to be required when page @@ -8539,34 +9105,20 @@ static int intel_gen6_queue_flip(struct drm_device *dev, intel_mark_page_flip_active(intel_crtc); __intel_ring_advance(ring); return 0; - -err_unpin: - intel_unpin_fb_obj(obj); -err: - return ret; } static int intel_gen7_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_ring_buffer *ring; uint32_t plane_bit = 0; int len, ret; - ring = obj->ring; - if (IS_VALLEYVIEW(dev) || ring == NULL || ring->id != RCS) - ring = &dev_priv->ring[BCS]; - - ret = intel_pin_and_fence_fb_obj(dev, obj, ring); - if (ret) - goto err; - - switch(intel_crtc->plane) { + switch (intel_crtc->plane) { case PLANE_A: plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A; break; @@ -8578,17 +9130,38 @@ static int intel_gen7_queue_flip(struct drm_device *dev, break; default: WARN_ONCE(1, "unknown plane in flip command\n"); - ret = -ENODEV; - goto err_unpin; + return -ENODEV; } len = 4; - if (ring->id == RCS) + if (ring->id == RCS) { len += 6; + /* + * On Gen 8, SRM is now taking an extra dword to accommodate + * 48bits addresses, and we need a NOOP for the batch size to + * stay even. + */ + if (IS_GEN8(dev)) + len += 2; + } + + /* + * BSpec MI_DISPLAY_FLIP for IVB: + * "The full packet must be contained within the same cache line." + * + * Currently the LRI+SRM+MI_DISPLAY_FLIP all fit within the same + * cacheline, if we ever start emitting more commands before + * the MI_DISPLAY_FLIP we may need to first emit everything else, + * then do the cacheline alignment, and finally emit the + * MI_DISPLAY_FLIP. + */ + ret = intel_ring_cacheline_align(ring); + if (ret) + return ret; ret = intel_ring_begin(ring, len); if (ret) - goto err_unpin; + return ret; /* Unmask the flip-done completion message. Note that the bspec says that * we should do this for both the BCS and RCS, and that we must not unmask @@ -8605,31 +9178,35 @@ static int intel_gen7_queue_flip(struct drm_device *dev, intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE | DERRMR_PIPEB_PRI_FLIP_DONE | DERRMR_PIPEC_PRI_FLIP_DONE)); - intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | - MI_SRM_LRM_GLOBAL_GTT); + if (IS_GEN8(dev)) + intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8(1) | + MI_SRM_LRM_GLOBAL_GTT); + else + intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | + MI_SRM_LRM_GLOBAL_GTT); intel_ring_emit(ring, DERRMR); intel_ring_emit(ring, ring->scratch.gtt_offset + 256); + if (IS_GEN8(dev)) { + intel_ring_emit(ring, 0); + intel_ring_emit(ring, MI_NOOP); + } } intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit); intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode)); - intel_ring_emit(ring, i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset); + intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset); intel_ring_emit(ring, (MI_NOOP)); intel_mark_page_flip_active(intel_crtc); __intel_ring_advance(ring); return 0; - -err_unpin: - intel_unpin_fb_obj(obj); -err: - return ret; } static int intel_default_queue_flip(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, + struct intel_engine_cs *ring, uint32_t flags) { return -ENODEV; @@ -8642,15 +9219,16 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *old_fb = crtc->fb; + struct drm_framebuffer *old_fb = crtc->primary->fb; struct drm_i915_gem_object *obj = to_intel_framebuffer(fb)->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; + struct intel_engine_cs *ring; unsigned long flags; int ret; /* Can't change pixel format via MI display flips. */ - if (fb->pixel_format != crtc->fb->pixel_format) + if (fb->pixel_format != crtc->primary->fb->pixel_format) return -EINVAL; /* @@ -8658,10 +9236,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, * Note that pitch changes could also affect these register. */ if (INTEL_INFO(dev)->gen > 3 && - (fb->offsets[0] != crtc->fb->offsets[0] || - fb->pitches[0] != crtc->fb->pitches[0])) + (fb->offsets[0] != crtc->primary->fb->offsets[0] || + fb->pitches[0] != crtc->primary->fb->pitches[0])) return -EINVAL; + if (i915_terminally_wedged(&dev_priv->gpu_error)) + goto out_hang; + work = kzalloc(sizeof(*work), GFP_KERNEL); if (work == NULL) return -ENOMEM; @@ -8671,7 +9252,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, work->old_fb_obj = to_intel_framebuffer(old_fb)->obj; INIT_WORK(&work->work, intel_unpin_work_fn); - ret = drm_vblank_get(dev, intel_crtc->pipe); + ret = drm_crtc_vblank_get(crtc); if (ret) goto free_work; @@ -8680,7 +9261,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, if (intel_crtc->unpin_work) { spin_unlock_irqrestore(&dev->event_lock, flags); kfree(work); - drm_vblank_put(dev, intel_crtc->pipe); + drm_crtc_vblank_put(crtc); DRM_DEBUG_DRIVER("flip queue: crtc already busy\n"); return -EBUSY; @@ -8699,7 +9280,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, drm_gem_object_reference(&work->old_fb_obj->base); drm_gem_object_reference(&obj->base); - crtc->fb = fb; + crtc->primary->fb = fb; work->pending_flip_obj = obj; @@ -8708,10 +9289,30 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, atomic_inc(&intel_crtc->unpin_work_count); intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); - ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, page_flip_flags); + if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) + work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(intel_crtc->pipe)) + 1; + + if (IS_VALLEYVIEW(dev)) { + ring = &dev_priv->ring[BCS]; + } else if (INTEL_INFO(dev)->gen >= 7) { + ring = obj->ring; + if (ring == NULL || ring->id != RCS) + ring = &dev_priv->ring[BCS]; + } else { + ring = &dev_priv->ring[RCS]; + } + + ret = intel_pin_and_fence_fb_obj(dev, obj, ring); if (ret) goto cleanup_pending; + work->gtt_offset = + i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset; + + ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring, page_flip_flags); + if (ret) + goto cleanup_unpin; + intel_disable_fbc(dev); intel_mark_fb_busy(obj, NULL); mutex_unlock(&dev->struct_mutex); @@ -8720,9 +9321,11 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, return 0; +cleanup_unpin: + intel_unpin_fb_obj(obj); cleanup_pending: atomic_dec(&intel_crtc->unpin_work_count); - crtc->fb = old_fb; + crtc->primary->fb = old_fb; drm_gem_object_unreference(&work->old_fb_obj->base); drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); @@ -8732,10 +9335,17 @@ cleanup: intel_crtc->unpin_work = NULL; spin_unlock_irqrestore(&dev->event_lock, flags); - drm_vblank_put(dev, intel_crtc->pipe); + drm_crtc_vblank_put(crtc); free_work: kfree(work); + if (ret == -EIO) { +out_hang: + intel_crtc_wait_for_pending_flips(crtc); + ret = intel_pipe_set_base(crtc, crtc->x, crtc->y, fb); + if (ret == 0 && event) + drm_send_vblank_event(dev, intel_crtc->pipe, event); + } return ret; } @@ -8752,6 +9362,7 @@ static struct drm_crtc_helper_funcs intel_helper_funcs = { */ static void intel_modeset_update_staged_output_state(struct drm_device *dev) { + struct intel_crtc *crtc; struct intel_encoder *encoder; struct intel_connector *connector; @@ -8766,6 +9377,15 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev) encoder->new_crtc = to_intel_crtc(encoder->base.crtc); } + + for_each_intel_crtc(dev, crtc) { + crtc->new_enabled = crtc->base.enabled; + + if (crtc->new_enabled) + crtc->new_config = &crtc->config; + else + crtc->new_config = NULL; + } } /** @@ -8775,6 +9395,7 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev) */ static void intel_modeset_commit_output_state(struct drm_device *dev) { + struct intel_crtc *crtc; struct intel_encoder *encoder; struct intel_connector *connector; @@ -8787,17 +9408,21 @@ static void intel_modeset_commit_output_state(struct drm_device *dev) base.head) { encoder->base.crtc = &encoder->new_crtc->base; } + + for_each_intel_crtc(dev, crtc) { + crtc->base.enabled = crtc->new_enabled; + } } static void -connected_sink_compute_bpp(struct intel_connector * connector, +connected_sink_compute_bpp(struct intel_connector *connector, struct intel_crtc_config *pipe_config) { int bpp = pipe_config->pipe_bpp; DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n", connector->base.base.id, - drm_get_connector_name(&connector->base)); + connector->base.name); /* Don't use an invalid EDID bpc value */ if (connector->base.display_info.bpc && @@ -8927,23 +9552,47 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide); } -static bool check_encoder_cloning(struct drm_crtc *crtc) +static bool encoders_cloneable(const struct intel_encoder *a, + const struct intel_encoder *b) +{ + /* masks could be asymmetric, so check both ways */ + return a == b || (a->cloneable & (1 << b->type) && + b->cloneable & (1 << a->type)); +} + +static bool check_single_encoder_cloning(struct intel_crtc *crtc, + struct intel_encoder *encoder) +{ + struct drm_device *dev = crtc->base.dev; + struct intel_encoder *source_encoder; + + list_for_each_entry(source_encoder, + &dev->mode_config.encoder_list, base.head) { + if (source_encoder->new_crtc != crtc) + continue; + + if (!encoders_cloneable(encoder, source_encoder)) + return false; + } + + return true; +} + +static bool check_encoder_cloning(struct intel_crtc *crtc) { - int num_encoders = 0; - bool uncloneable_encoders = false; + struct drm_device *dev = crtc->base.dev; struct intel_encoder *encoder; - list_for_each_entry(encoder, &crtc->dev->mode_config.encoder_list, - base.head) { - if (&encoder->new_crtc->base != crtc) + list_for_each_entry(encoder, + &dev->mode_config.encoder_list, base.head) { + if (encoder->new_crtc != crtc) continue; - num_encoders++; - if (!encoder->cloneable) - uncloneable_encoders = true; + if (!check_single_encoder_cloning(crtc, encoder)) + return false; } - return !(num_encoders > 1 && uncloneable_encoders); + return true; } static struct intel_crtc_config * @@ -8957,7 +9606,7 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, int plane_bpp, ret = -EINVAL; bool retry = true; - if (!check_encoder_cloning(crtc)) { + if (!check_encoder_cloning(to_intel_crtc(crtc))) { DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); return ERR_PTR(-EINVAL); } @@ -9113,29 +9762,21 @@ intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes, *prepare_pipes |= 1 << encoder->new_crtc->pipe; } - /* Check for any pipes that will be fully disabled ... */ - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, - base.head) { - bool used = false; - - /* Don't try to disable disabled crtcs. */ - if (!intel_crtc->base.enabled) + /* Check for pipes that will be enabled/disabled ... */ + for_each_intel_crtc(dev, intel_crtc) { + if (intel_crtc->base.enabled == intel_crtc->new_enabled) continue; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, - base.head) { - if (encoder->new_crtc == intel_crtc) - used = true; - } - - if (!used) + if (!intel_crtc->new_enabled) *disable_pipes |= 1 << intel_crtc->pipe; + else + *prepare_pipes |= 1 << intel_crtc->pipe; } /* set_mode is also used to update properties on life display pipes. */ intel_crtc = to_intel_crtc(crtc); - if (crtc->enabled) + if (intel_crtc->new_enabled) *prepare_pipes |= 1 << intel_crtc->pipe; /* @@ -9194,10 +9835,12 @@ intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes) intel_modeset_commit_output_state(dev); - /* Update computed state. */ - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, - base.head) { - intel_crtc->base.enabled = intel_crtc_in_use(&intel_crtc->base); + /* Double check state. */ + for_each_intel_crtc(dev, intel_crtc) { + WARN_ON(intel_crtc->base.enabled != intel_crtc_in_use(&intel_crtc->base)); + WARN_ON(intel_crtc->new_config && + intel_crtc->new_config != &intel_crtc->config); + WARN_ON(intel_crtc->base.enabled != !!intel_crtc->new_config); } list_for_each_entry(connector, &dev->mode_config.connector_list, head) { @@ -9322,6 +9965,12 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_I(adjusted_mode.crtc_vsync_end); PIPE_CONF_CHECK_I(pixel_multiplier); + PIPE_CONF_CHECK_I(has_hdmi_sink); + if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) || + IS_VALLEYVIEW(dev)) + PIPE_CONF_CHECK_I(limited_color_range); + + PIPE_CONF_CHECK_I(has_audio); PIPE_CONF_CHECK_FLAGS(adjusted_mode.flags, DRM_MODE_FLAG_INTERLACE); @@ -9340,11 +9989,22 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_I(pipe_src_w); PIPE_CONF_CHECK_I(pipe_src_h); - PIPE_CONF_CHECK_I(gmch_pfit.control); - /* pfit ratios are autocomputed by the hw on gen4+ */ - if (INTEL_INFO(dev)->gen < 4) - PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios); - PIPE_CONF_CHECK_I(gmch_pfit.lvds_border_bits); + /* + * FIXME: BIOS likes to set up a cloned config with lvds+external + * screen. Since we don't yet re-compute the pipe config when moving + * just the lvds port away to another pipe the sw tracking won't match. + * + * Proper atomic modesets with recomputed global state will fix this. + * Until then just don't check gmch state for inherited modes. + */ + if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_INHERITED_MODE)) { + PIPE_CONF_CHECK_I(gmch_pfit.control); + /* pfit ratios are autocomputed by the hw on gen4+ */ + if (INTEL_INFO(dev)->gen < 4) + PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios); + PIPE_CONF_CHECK_I(gmch_pfit.lvds_border_bits); + } + PIPE_CONF_CHECK_I(pch_pfit.enabled); if (current_config->pch_pfit.enabled) { PIPE_CONF_CHECK_I(pch_pfit.pos); @@ -9366,10 +10026,8 @@ intel_pipe_config_compare(struct drm_device *dev, if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) PIPE_CONF_CHECK_I(pipe_bpp); - if (!HAS_DDI(dev)) { - PIPE_CONF_CHECK_CLOCK_FUZZY(adjusted_mode.crtc_clock); - PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock); - } + PIPE_CONF_CHECK_CLOCK_FUZZY(adjusted_mode.crtc_clock); + PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock); #undef PIPE_CONF_CHECK_X #undef PIPE_CONF_CHECK_I @@ -9410,7 +10068,7 @@ check_encoder_state(struct drm_device *dev) DRM_DEBUG_KMS("[ENCODER:%d:%s]\n", encoder->base.base.id, - drm_get_encoder_name(&encoder->base)); + encoder->base.name); WARN(&encoder->new_crtc->base != encoder->base.crtc, "encoder's stage crtc doesn't match current crtc\n"); @@ -9457,13 +10115,12 @@ check_encoder_state(struct drm_device *dev) static void check_crtc_state(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc; struct intel_encoder *encoder; struct intel_crtc_config pipe_config; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, - base.head) { + for_each_intel_crtc(dev, crtc) { bool enabled = false; bool active = false; @@ -9525,7 +10182,7 @@ check_crtc_state(struct drm_device *dev) static void check_shared_dpll_state(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *crtc; struct intel_dpll_hw_state dpll_hw_state; int i; @@ -9552,8 +10209,7 @@ check_shared_dpll_state(struct drm_device *dev) "pll on state mismatch (expected %i, found %i)\n", pll->on, active); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, - base.head) { + for_each_intel_crtc(dev, crtc) { if (crtc->base.enabled && intel_crtc_to_shared_dpll(crtc) == pll) enabled_crtcs++; if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll) @@ -9593,12 +10249,50 @@ void ironlake_check_encoder_dotclock(const struct intel_crtc_config *pipe_config pipe_config->adjusted_mode.crtc_clock, dotclock); } +static void update_scanline_offset(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + + /* + * The scanline counter increments at the leading edge of hsync. + * + * On most platforms it starts counting from vtotal-1 on the + * first active line. That means the scanline counter value is + * always one less than what we would expect. Ie. just after + * start of vblank, which also occurs at start of hsync (on the + * last active line), the scanline counter will read vblank_start-1. + * + * On gen2 the scanline counter starts counting from 1 instead + * of vtotal-1, so we have to subtract one (or rather add vtotal-1 + * to keep the value positive), instead of adding one. + * + * On HSW+ the behaviour of the scanline counter depends on the output + * type. For DP ports it behaves like most other platforms, but on HDMI + * there's an extra 1 line difference. So we need to add two instead of + * one to the value. + */ + if (IS_GEN2(dev)) { + const struct drm_display_mode *mode = &crtc->config.adjusted_mode; + int vtotal; + + vtotal = mode->crtc_vtotal; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + vtotal /= 2; + + crtc->scanline_offset = vtotal - 1; + } else if (HAS_DDI(dev) && + intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_HDMI)) { + crtc->scanline_offset = 2; + } else + crtc->scanline_offset = 1; +} + static int __intel_set_mode(struct drm_crtc *crtc, struct drm_display_mode *mode, int x, int y, struct drm_framebuffer *fb) { struct drm_device *dev = crtc->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_display_mode *saved_mode; struct intel_crtc_config *pipe_config = NULL; struct intel_crtc *intel_crtc; @@ -9629,6 +10323,7 @@ static int __intel_set_mode(struct drm_crtc *crtc, } intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config, "[modeset]"); + to_intel_crtc(crtc)->new_config = pipe_config; } /* @@ -9639,8 +10334,7 @@ static int __intel_set_mode(struct drm_crtc *crtc, * adjusted_mode bits in the crtc directly. */ if (IS_VALLEYVIEW(dev)) { - valleyview_modeset_global_pipes(dev, &prepare_pipes, - modeset_pipes, pipe_config); + valleyview_modeset_global_pipes(dev, &prepare_pipes); /* may have added more to prepare_pipes than we should */ prepare_pipes &= ~disable_pipes; @@ -9662,6 +10356,7 @@ static int __intel_set_mode(struct drm_crtc *crtc, /* mode_set/enable/disable functions rely on a correct pipe * config. */ to_intel_crtc(crtc)->config = *pipe_config; + to_intel_crtc(crtc)->new_config = &to_intel_crtc(crtc)->config; /* * Calculate and store various constants which @@ -9683,15 +10378,38 @@ static int __intel_set_mode(struct drm_crtc *crtc, * on the DPLL. */ for_each_intel_crtc_masked(dev, modeset_pipes, intel_crtc) { - ret = intel_crtc_mode_set(&intel_crtc->base, - x, y, fb); + struct drm_framebuffer *old_fb; + + mutex_lock(&dev->struct_mutex); + ret = intel_pin_and_fence_fb_obj(dev, + to_intel_framebuffer(fb)->obj, + NULL); + if (ret != 0) { + DRM_ERROR("pin & fence failed\n"); + mutex_unlock(&dev->struct_mutex); + goto done; + } + old_fb = crtc->primary->fb; + if (old_fb) + intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj); + mutex_unlock(&dev->struct_mutex); + + crtc->primary->fb = fb; + crtc->x = x; + crtc->y = y; + + ret = dev_priv->display.crtc_mode_set(&intel_crtc->base, + x, y, fb); if (ret) goto done; } /* Now enable the clocks, plane, pipe, and connectors that we set up. */ - for_each_intel_crtc_masked(dev, prepare_pipes, intel_crtc) + for_each_intel_crtc_masked(dev, prepare_pipes, intel_crtc) { + update_scanline_offset(intel_crtc); + dev_priv->display.crtc_enable(&intel_crtc->base); + } /* FIXME: add subpixel order */ done: @@ -9720,7 +10438,7 @@ static int intel_set_mode(struct drm_crtc *crtc, void intel_crtc_restore_mode(struct drm_crtc *crtc) { - intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y, crtc->fb); + intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y, crtc->primary->fb); } #undef for_each_intel_crtc_masked @@ -9732,16 +10450,24 @@ static void intel_set_config_free(struct intel_set_config *config) kfree(config->save_connector_encoders); kfree(config->save_encoder_crtcs); + kfree(config->save_crtc_enabled); kfree(config); } static int intel_set_config_save_state(struct drm_device *dev, struct intel_set_config *config) { + struct drm_crtc *crtc; struct drm_encoder *encoder; struct drm_connector *connector; int count; + config->save_crtc_enabled = + kcalloc(dev->mode_config.num_crtc, + sizeof(bool), GFP_KERNEL); + if (!config->save_crtc_enabled) + return -ENOMEM; + config->save_encoder_crtcs = kcalloc(dev->mode_config.num_encoder, sizeof(struct drm_crtc *), GFP_KERNEL); @@ -9759,6 +10485,11 @@ static int intel_set_config_save_state(struct drm_device *dev, * restored, not the drivers personal bookkeeping. */ count = 0; + for_each_crtc(dev, crtc) { + config->save_crtc_enabled[count++] = crtc->enabled; + } + + count = 0; list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { config->save_encoder_crtcs[count++] = encoder->crtc; } @@ -9774,11 +10505,22 @@ static int intel_set_config_save_state(struct drm_device *dev, static void intel_set_config_restore_state(struct drm_device *dev, struct intel_set_config *config) { + struct intel_crtc *crtc; struct intel_encoder *encoder; struct intel_connector *connector; int count; count = 0; + for_each_intel_crtc(dev, crtc) { + crtc->new_enabled = config->save_crtc_enabled[count++]; + + if (crtc->new_enabled) + crtc->new_config = &crtc->config; + else + crtc->new_config = NULL; + } + + count = 0; list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { encoder->new_crtc = to_intel_crtc(config->save_encoder_crtcs[count++]); @@ -9820,13 +10562,13 @@ intel_set_config_compute_mode_changes(struct drm_mode_set *set, * and then just flip_or_move it */ if (is_crtc_connector_off(set)) { config->mode_changed = true; - } else if (set->crtc->fb != set->fb) { + } else if (set->crtc->primary->fb != set->fb) { /* If we have no fb then treat it as a full mode set */ - if (set->crtc->fb == NULL) { + if (set->crtc->primary->fb == NULL) { struct intel_crtc *intel_crtc = to_intel_crtc(set->crtc); - if (intel_crtc->active && i915_fastboot) { + if (intel_crtc->active && i915.fastboot) { DRM_DEBUG_KMS("crtc has no fb, will flip\n"); config->fb_changed = true; } else { @@ -9836,7 +10578,7 @@ intel_set_config_compute_mode_changes(struct drm_mode_set *set, } else if (set->fb == NULL) { config->mode_changed = true; } else if (set->fb->pixel_format != - set->crtc->fb->pixel_format) { + set->crtc->primary->fb->pixel_format) { config->mode_changed = true; } else { config->fb_changed = true; @@ -9862,9 +10604,9 @@ intel_modeset_stage_output_state(struct drm_device *dev, struct drm_mode_set *set, struct intel_set_config *config) { - struct drm_crtc *new_crtc; struct intel_connector *connector; struct intel_encoder *encoder; + struct intel_crtc *crtc; int ro; /* The upper layers ensure that we either disable a crtc or have a list @@ -9893,7 +10635,7 @@ intel_modeset_stage_output_state(struct drm_device *dev, DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [NOCRTC]\n", connector->base.base.id, - drm_get_connector_name(&connector->base)); + connector->base.name); } @@ -9907,6 +10649,8 @@ intel_modeset_stage_output_state(struct drm_device *dev, /* Update crtc of enabled connectors. */ list_for_each_entry(connector, &dev->mode_config.connector_list, base.head) { + struct drm_crtc *new_crtc; + if (!connector->new_encoder) continue; @@ -9926,7 +10670,7 @@ intel_modeset_stage_output_state(struct drm_device *dev, DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d]\n", connector->base.base.id, - drm_get_connector_name(&connector->base), + connector->base.name, new_crtc->base.id); } @@ -9957,9 +10701,57 @@ intel_modeset_stage_output_state(struct drm_device *dev, } /* Now we've also updated encoder->new_crtc for all encoders. */ + for_each_intel_crtc(dev, crtc) { + crtc->new_enabled = false; + + list_for_each_entry(encoder, + &dev->mode_config.encoder_list, + base.head) { + if (encoder->new_crtc == crtc) { + crtc->new_enabled = true; + break; + } + } + + if (crtc->new_enabled != crtc->base.enabled) { + DRM_DEBUG_KMS("crtc %sabled, full mode switch\n", + crtc->new_enabled ? "en" : "dis"); + config->mode_changed = true; + } + + if (crtc->new_enabled) + crtc->new_config = &crtc->config; + else + crtc->new_config = NULL; + } + return 0; } +static void disable_crtc_nofb(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct intel_encoder *encoder; + struct intel_connector *connector; + + DRM_DEBUG_KMS("Trying to restore without FB -> disabling pipe %c\n", + pipe_name(crtc->pipe)); + + list_for_each_entry(connector, &dev->mode_config.connector_list, base.head) { + if (connector->new_encoder && + connector->new_encoder->new_crtc == crtc) + connector->new_encoder = NULL; + } + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + if (encoder->new_crtc == crtc) + encoder->new_crtc = NULL; + } + + crtc->new_enabled = false; + crtc->new_config = NULL; +} + static int intel_crtc_set_config(struct drm_mode_set *set) { struct drm_device *dev; @@ -9998,7 +10790,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set) save_set.mode = &set->crtc->mode; save_set.x = set->crtc->x; save_set.y = set->crtc->y; - save_set.fb = set->crtc->fb; + save_set.fb = set->crtc->primary->fb; /* Compute whether we need a full modeset, only an fb base update or no * change at all. In the future we might also check whether only the @@ -10026,7 +10818,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set) * flipping, so increasing its cost here shouldn't be a big * deal). */ - if (i915_fastboot && ret == 0) + if (i915.fastboot && ret == 0) intel_modeset_check_state(set->crtc->dev); } @@ -10036,6 +10828,15 @@ static int intel_crtc_set_config(struct drm_mode_set *set) fail: intel_set_config_restore_state(dev, config); + /* + * HACK: if the pipe was on, but we didn't have a framebuffer, + * force the pipe off to avoid oopsing in the modeset code + * due to fb==NULL. This should only happen during boot since + * we don't yet reconstruct the FB from the hardware state. + */ + if (to_intel_crtc(save_set.crtc)->new_enabled && !save_set.fb) + disable_crtc_nofb(to_intel_crtc(save_set.crtc)); + /* Try to restore the config */ if (config->mode_changed && intel_set_mode(save_set.crtc, save_set.mode, @@ -10113,7 +10914,7 @@ static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv, struct intel_crtc *crtc; /* Make sure no transcoder isn't still depending on us. */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) { + for_each_intel_crtc(dev, crtc) { if (intel_crtc_to_shared_dpll(crtc) == pll) assert_pch_transcoder_disabled(dev_priv, crtc->pipe); } @@ -10160,7 +10961,7 @@ static void intel_shared_dpll_init(struct drm_device *dev) static void intel_crtc_init(struct drm_device *dev, int pipe) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc; int i; @@ -10188,19 +10989,27 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) intel_crtc->plane = !pipe; } + intel_crtc->cursor_base = ~0; + intel_crtc->cursor_cntl = ~0; + + init_waitqueue_head(&intel_crtc->vbl_wait); + BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL); dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base; drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); + + WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe); } enum pipe intel_get_pipe_from_connector(struct intel_connector *connector) { struct drm_encoder *encoder = connector->base.encoder; + struct drm_device *dev = connector->base.dev; - WARN_ON(!mutex_is_locked(&connector->base.dev->mode_config.mutex)); + WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); if (!encoder) return INVALID_PIPE; @@ -10241,12 +11050,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder) list_for_each_entry(source_encoder, &dev->mode_config.encoder_list, base.head) { - - if (encoder == source_encoder) - index_mask |= (1 << entry); - - /* Intel hw has only one MUX where enocoders could be cloned. */ - if (encoder->cloneable && source_encoder->cloneable) + if (encoders_cloneable(encoder, source_encoder)) index_mask |= (1 << entry); entry++; @@ -10265,8 +11069,7 @@ static bool has_edp_a(struct drm_device *dev) if ((I915_READ(DP_A) & DP_DETECTED) == 0) return false; - if (IS_GEN5(dev) && - (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE)) + if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE)) return false; return true; @@ -10294,6 +11097,22 @@ const char *intel_output_name(int output) return names[output]; } +static bool intel_crt_present(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (IS_ULT(dev)) + return false; + + if (IS_CHERRYVIEW(dev)) + return false; + + if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support) + return false; + + return true; +} + static void intel_setup_outputs(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -10302,7 +11121,7 @@ static void intel_setup_outputs(struct drm_device *dev) intel_lvds_init(dev); - if (!IS_ULT(dev)) + if (intel_crt_present(dev)) intel_crt_init(dev); if (HAS_DDI(dev)) { @@ -10366,6 +11185,15 @@ static void intel_setup_outputs(struct drm_device *dev) intel_dp_init(dev, VLV_DISPLAY_BASE + DP_C, PORT_C); } + if (IS_CHERRYVIEW(dev)) { + if (I915_READ(VLV_DISPLAY_BASE + CHV_HDMID) & SDVO_DETECTED) { + intel_hdmi_init(dev, VLV_DISPLAY_BASE + CHV_HDMID, + PORT_D); + if (I915_READ(VLV_DISPLAY_BASE + DP_D) & DP_DETECTED) + intel_dp_init(dev, VLV_DISPLAY_BASE + DP_D, PORT_D); + } + } + intel_dsi_init(dev); } else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) { bool found = false; @@ -10419,18 +11247,13 @@ static void intel_setup_outputs(struct drm_device *dev) drm_helper_move_panel_connectors_to_head(dev); } -void intel_framebuffer_fini(struct intel_framebuffer *fb) -{ - drm_framebuffer_cleanup(&fb->base); - WARN_ON(!fb->obj->framebuffer_references--); - drm_gem_object_unreference_unlocked(&fb->obj->base); -} - static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) { struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - intel_framebuffer_fini(intel_fb); + drm_framebuffer_cleanup(fb); + WARN_ON(!intel_fb->obj->framebuffer_references--); + drm_gem_object_unreference_unlocked(&intel_fb->obj->base); kfree(intel_fb); } @@ -10449,12 +11272,12 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = { .create_handle = intel_user_framebuffer_create_handle, }; -int intel_framebuffer_init(struct drm_device *dev, - struct intel_framebuffer *intel_fb, - struct drm_mode_fb_cmd2 *mode_cmd, - struct drm_i915_gem_object *obj) +static int intel_framebuffer_init(struct drm_device *dev, + struct intel_framebuffer *intel_fb, + struct drm_mode_fb_cmd2 *mode_cmd, + struct drm_i915_gem_object *obj) { - int aligned_height, tile_height; + int aligned_height; int pitch_limit; int ret; @@ -10548,9 +11371,8 @@ int intel_framebuffer_init(struct drm_device *dev, if (mode_cmd->offsets[0] != 0) return -EINVAL; - tile_height = IS_GEN2(dev) ? 16 : 8; - aligned_height = ALIGN(mode_cmd->height, - obj->tiling_mode ? tile_height : 1); + aligned_height = intel_align_height(dev, mode_cmd->height, + obj->tiling_mode); /* FIXME drm helper for size checks (especially planar formats)? */ if (obj->base.size < aligned_height * mode_cmd->pitches[0]) return -EINVAL; @@ -10601,6 +11423,8 @@ static void intel_init_display(struct drm_device *dev) if (HAS_PCH_SPLIT(dev) || IS_G4X(dev)) dev_priv->display.find_dpll = g4x_find_best_dpll; + else if (IS_CHERRYVIEW(dev)) + dev_priv->display.find_dpll = chv_find_best_dpll; else if (IS_VALLEYVIEW(dev)) dev_priv->display.find_dpll = vlv_find_best_dpll; else if (IS_PINEVIEW(dev)) @@ -10610,32 +11434,40 @@ static void intel_init_display(struct drm_device *dev) if (HAS_DDI(dev)) { dev_priv->display.get_pipe_config = haswell_get_pipe_config; + dev_priv->display.get_plane_config = ironlake_get_plane_config; dev_priv->display.crtc_mode_set = haswell_crtc_mode_set; dev_priv->display.crtc_enable = haswell_crtc_enable; dev_priv->display.crtc_disable = haswell_crtc_disable; dev_priv->display.off = haswell_crtc_off; - dev_priv->display.update_plane = ironlake_update_plane; + dev_priv->display.update_primary_plane = + ironlake_update_primary_plane; } else if (HAS_PCH_SPLIT(dev)) { dev_priv->display.get_pipe_config = ironlake_get_pipe_config; + dev_priv->display.get_plane_config = ironlake_get_plane_config; dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set; dev_priv->display.crtc_enable = ironlake_crtc_enable; dev_priv->display.crtc_disable = ironlake_crtc_disable; dev_priv->display.off = ironlake_crtc_off; - dev_priv->display.update_plane = ironlake_update_plane; + dev_priv->display.update_primary_plane = + ironlake_update_primary_plane; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.get_pipe_config = i9xx_get_pipe_config; + dev_priv->display.get_plane_config = i9xx_get_plane_config; dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set; dev_priv->display.crtc_enable = valleyview_crtc_enable; dev_priv->display.crtc_disable = i9xx_crtc_disable; dev_priv->display.off = i9xx_crtc_off; - dev_priv->display.update_plane = i9xx_update_plane; + dev_priv->display.update_primary_plane = + i9xx_update_primary_plane; } else { dev_priv->display.get_pipe_config = i9xx_get_pipe_config; + dev_priv->display.get_plane_config = i9xx_get_plane_config; dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set; dev_priv->display.crtc_enable = i9xx_crtc_enable; dev_priv->display.crtc_disable = i9xx_crtc_disable; dev_priv->display.off = i9xx_crtc_off; - dev_priv->display.update_plane = i9xx_update_plane; + dev_priv->display.update_primary_plane = + i9xx_update_primary_plane; } /* Returns the core display clock speed */ @@ -10674,6 +11506,8 @@ static void intel_init_display(struct drm_device *dev) } else if (IS_GEN6(dev)) { dev_priv->display.fdi_link_train = gen6_fdi_link_train; dev_priv->display.write_eld = ironlake_write_eld; + dev_priv->display.modeset_global_resources = + snb_modeset_global_resources; } else if (IS_IVYBRIDGE(dev)) { /* FIXME: detect B0+ stepping and use auto training */ dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; @@ -10757,6 +11591,14 @@ static void quirk_invert_brightness(struct drm_device *dev) DRM_INFO("applying inverted panel brightness quirk\n"); } +/* Some VBT's incorrectly indicate no backlight is present */ +static void quirk_backlight_present(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT; + DRM_INFO("applying backlight present quirk\n"); +} + struct intel_quirk { int device; int subsystem_vendor; @@ -10802,9 +11644,6 @@ static struct intel_quirk intel_quirks[] = { /* ThinkPad T60 needs pipe A force quirk (bug #16494) */ { 0x2782, 0x17aa, 0x201a, quirk_pipea_force }, - /* 830 needs to leave pipe A & dpll A up */ - { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force }, - /* Lenovo U160 cannot use SSC on LVDS */ { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable }, @@ -10825,6 +11664,18 @@ static struct intel_quirk intel_quirks[] = { /* Acer Aspire 4736Z */ { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness }, + + /* Acer Aspire 5336 */ + { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness }, + + /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */ + { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present }, + + /* Toshiba CB35 Chromebook (Celeron 2955U) */ + { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present }, + + /* HP Chromebook 14 (Celeron 2955U) */ + { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present }, }; static void intel_init_quirks(struct drm_device *dev) @@ -10855,6 +11706,7 @@ static void i915_disable_vga(struct drm_device *dev) u8 sr1; u32 vga_reg = i915_vgacntrl_reg(dev); + /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */ vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO); outb(SR01, VGA_SR_INDEX); sr1 = inb(VGA_SR_DATA); @@ -10874,9 +11726,7 @@ void intel_modeset_init_hw(struct drm_device *dev) intel_reset_dpio(dev); - mutex_lock(&dev->struct_mutex); intel_enable_gt_powersave(dev); - mutex_unlock(&dev->struct_mutex); } void intel_modeset_suspend_hw(struct drm_device *dev) @@ -10887,7 +11737,9 @@ void intel_modeset_suspend_hw(struct drm_device *dev) void intel_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - int i, j, ret; + int sprite, ret; + enum pipe pipe; + struct intel_crtc *crtc; drm_mode_config_init(dev); @@ -10918,19 +11770,28 @@ void intel_modeset_init(struct drm_device *dev) dev->mode_config.max_width = 8192; dev->mode_config.max_height = 8192; } + + if (IS_GEN2(dev)) { + dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH; + dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT; + } else { + dev->mode_config.cursor_width = MAX_CURSOR_WIDTH; + dev->mode_config.cursor_height = MAX_CURSOR_HEIGHT; + } + dev->mode_config.fb_base = dev_priv->gtt.mappable_base; DRM_DEBUG_KMS("%d display pipe%s available.\n", INTEL_INFO(dev)->num_pipes, INTEL_INFO(dev)->num_pipes > 1 ? "s" : ""); - for_each_pipe(i) { - intel_crtc_init(dev, i); - for (j = 0; j < dev_priv->num_plane; j++) { - ret = intel_plane_init(dev, i, j); + for_each_pipe(pipe) { + intel_crtc_init(dev, pipe); + for_each_sprite(pipe, sprite) { + ret = intel_plane_init(dev, pipe, sprite); if (ret) DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n", - pipe_name(i), sprite_name(i, j), ret); + pipe_name(pipe), sprite_name(pipe, sprite), ret); } } @@ -10946,15 +11807,32 @@ void intel_modeset_init(struct drm_device *dev) /* Just in case the BIOS is doing something questionable. */ intel_disable_fbc(dev); -} -static void -intel_connector_break_all_links(struct intel_connector *connector) -{ - connector->base.dpms = DRM_MODE_DPMS_OFF; - connector->base.encoder = NULL; - connector->encoder->connectors_active = false; - connector->encoder->base.crtc = NULL; + drm_modeset_lock_all(dev); + intel_modeset_setup_hw_state(dev, false); + drm_modeset_unlock_all(dev); + + for_each_intel_crtc(dev, crtc) { + if (!crtc->active) + continue; + + /* + * Note that reserving the BIOS fb up front prevents us + * from stuffing other stolen allocations like the ring + * on top. This prevents some ugliness at boot time, and + * can even allow for smooth boot transitions if the BIOS + * fb is large enough for the active pipe configuration. + */ + if (dev_priv->display.get_plane_config) { + dev_priv->display.get_plane_config(crtc, + &crtc->plane_config); + /* + * If the fb is shared between multiple heads, we'll + * just get the first one. + */ + intel_find_plane_obj(crtc, &crtc->plane_config); + } + } } static void intel_enable_pipe_a(struct drm_device *dev) @@ -10962,6 +11840,7 @@ static void intel_enable_pipe_a(struct drm_device *dev) struct intel_connector *connector; struct drm_connector *crt = NULL; struct intel_load_detect_pipe load_detect_temp; + struct drm_modeset_acquire_ctx ctx; /* We can't just switch on the pipe A, we need to set things up with a * proper mode and output configuration. As a gross hack, enable pipe A @@ -10978,8 +11857,8 @@ static void intel_enable_pipe_a(struct drm_device *dev) if (!crt) return; - if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp)) - intel_release_load_detect_pipe(crt, &load_detect_temp); + if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp, &ctx)) + intel_release_load_detect_pipe(crt, &load_detect_temp, &ctx); } @@ -11014,6 +11893,12 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) reg = PIPECONF(crtc->config.cpu_transcoder); I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK); + /* restore vblank interrupts to correct state */ + if (crtc->active) + drm_vblank_on(dev, crtc->pipe); + else + drm_vblank_off(dev, crtc->pipe); + /* We need to sanitize the plane -> pipe mapping first because this will * disable the crtc (and hence change the state) if it is wrong. Note * that gen4+ has a fixed plane -> pipe mapping. */ @@ -11029,6 +11914,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) * ... */ plane = crtc->plane; crtc->plane = !plane; + crtc->primary_enabled = true; dev_priv->display.crtc_disable(&crtc->base); crtc->plane = plane; @@ -11038,8 +11924,17 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) if (connector->encoder->base.crtc != &crtc->base) continue; - intel_connector_break_all_links(connector); + connector->base.dpms = DRM_MODE_DPMS_OFF; + connector->base.encoder = NULL; } + /* multiple connectors may have the same encoder: + * handle them and break crtc link separately */ + list_for_each_entry(connector, &dev->mode_config.connector_list, + base.head) + if (connector->encoder->base.crtc == &crtc->base) { + connector->encoder->base.crtc = NULL; + connector->encoder->connectors_active = false; + } WARN_ON(crtc->active); crtc->base.enabled = false; @@ -11083,6 +11978,26 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) encoder->base.crtc = NULL; } } + + if (crtc->active || IS_VALLEYVIEW(dev) || INTEL_INFO(dev)->gen < 5) { + /* + * We start out with underrun reporting disabled to avoid races. + * For correct bookkeeping mark this on active crtcs. + * + * Also on gmch platforms we dont have any hardware bits to + * disable the underrun reporting. Which means we need to start + * out with underrun reporting disabled also on inactive pipes, + * since otherwise we'll complain about the garbage we read when + * e.g. coming up after runtime pm. + * + * No protection against concurrent access is required - at + * worst a fifo underrun happens which also sets this to false. + */ + crtc->cpu_fifo_underrun_disabled = true; + crtc->pch_fifo_underrun_disabled = true; + + update_scanline_offset(crtc); + } } static void intel_sanitize_encoder(struct intel_encoder *encoder) @@ -11099,7 +12014,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) if (encoder->connectors_active && !has_active_crtc) { DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n", encoder->base.base.id, - drm_get_encoder_name(&encoder->base)); + encoder->base.name); /* Connector is active, but has no active pipe. This is * fallout from our resume register restoring. Disable @@ -11107,9 +12022,11 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) if (encoder->base.crtc) { DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n", encoder->base.base.id, - drm_get_encoder_name(&encoder->base)); + encoder->base.name); encoder->disable(encoder); } + encoder->base.crtc = NULL; + encoder->connectors_active = false; /* Inconsistent output/port/pipe state happens presumably due to * a bug in one of the get_hw_state functions. Or someplace else @@ -11120,19 +12037,29 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) base.head) { if (connector->encoder != encoder) continue; - - intel_connector_break_all_links(connector); + connector->base.dpms = DRM_MODE_DPMS_OFF; + connector->base.encoder = NULL; } } /* Enabled encoders without active connectors will be fixed in * the crtc fixup. */ } -void i915_redisable_vga(struct drm_device *dev) +void i915_redisable_vga_power_on(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; u32 vga_reg = i915_vgacntrl_reg(dev); + if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { + DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n"); + i915_disable_vga(dev); + } +} + +void i915_redisable_vga(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + /* This function can be called both from intel_modeset_setup_hw_state or * at a very early point in our resume sequence, where the power well * structures are not yet restored. Since this function is at a very @@ -11140,14 +12067,20 @@ void i915_redisable_vga(struct drm_device *dev) * level, just check if the power well is enabled instead of trying to * follow the "don't touch the power well if we don't need it" policy * the rest of the driver uses. */ - if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && - (I915_READ(HSW_PWR_WELL_DRIVER) & HSW_PWR_WELL_STATE_ENABLED) == 0) + if (!intel_display_power_enabled(dev_priv, POWER_DOMAIN_VGA)) return; - if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { - DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n"); - i915_disable_vga(dev); - } + i915_redisable_vga_power_on(dev); +} + +static bool primary_get_hw_state(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + + if (!crtc->active) + return false; + + return I915_READ(DSPCNTR(crtc->plane)) & DISPLAY_PLANE_ENABLE; } static void intel_modeset_readout_hw_state(struct drm_device *dev) @@ -11159,15 +12092,16 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) struct intel_connector *connector; int i; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, - base.head) { + for_each_intel_crtc(dev, crtc) { memset(&crtc->config, 0, sizeof(crtc->config)); + crtc->config.quirks |= PIPE_CONFIG_QUIRK_INHERITED_MODE; + crtc->active = dev_priv->display.get_pipe_config(crtc, &crtc->config); crtc->base.enabled = crtc->active; - crtc->primary_enabled = crtc->active; + crtc->primary_enabled = primary_get_hw_state(crtc); DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n", crtc->base.base.id, @@ -11183,8 +12117,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) pll->on = pll->get_hw_state(dev_priv, pll, &pll->hw_state); pll->active = 0; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, - base.head) { + for_each_intel_crtc(dev, crtc) { if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll) pll->active++; } @@ -11209,7 +12142,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) encoder->connectors_active = false; DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n", encoder->base.base.id, - drm_get_encoder_name(&encoder->base), + encoder->base.name, encoder->base.crtc ? "enabled" : "disabled", pipe_name(pipe)); } @@ -11226,7 +12159,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) } DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n", connector->base.base.id, - drm_get_connector_name(&connector->base), + connector->base.name, connector->base.encoder ? "enabled" : "disabled"); } } @@ -11249,11 +12182,9 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, * Note that this could go away if we move to using crtc_config * checking everywhere. */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, - base.head) { - if (crtc->active && i915_fastboot) { - intel_crtc_mode_from_pipe_config(crtc, &crtc->config); - + for_each_intel_crtc(dev, crtc) { + if (crtc->active && i915.fastboot) { + intel_mode_from_pipe_config(&crtc->base.mode, &crtc->config); DRM_DEBUG_KMS("[CRTC:%d] found active mode: ", crtc->base.base.id); drm_mode_debug_printmodeline(&crtc->base.mode); @@ -11299,7 +12230,7 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, dev_priv->pipe_to_crtc_mapping[pipe]; __intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y, - crtc->fb); + crtc->primary->fb); } } else { intel_modeset_update_staged_output_state(dev); @@ -11310,14 +12241,44 @@ void intel_modeset_setup_hw_state(struct drm_device *dev, void intel_modeset_gem_init(struct drm_device *dev) { + struct drm_crtc *c; + struct intel_framebuffer *fb; + + mutex_lock(&dev->struct_mutex); + intel_init_gt_powersave(dev); + mutex_unlock(&dev->struct_mutex); + intel_modeset_init_hw(dev); intel_setup_overlay(dev); - mutex_lock(&dev->mode_config.mutex); - drm_mode_config_reset(dev); - intel_modeset_setup_hw_state(dev, false); - mutex_unlock(&dev->mode_config.mutex); + /* + * Make sure any fbs we allocated at startup are properly + * pinned & fenced. When we do the allocation it's too early + * for this. + */ + mutex_lock(&dev->struct_mutex); + for_each_crtc(dev, c) { + if (!c->primary->fb) + continue; + + fb = to_intel_framebuffer(c->primary->fb); + if (intel_pin_and_fence_fb_obj(dev, fb->obj, NULL)) { + DRM_ERROR("failed to pin boot fb on pipe %d\n", + to_intel_crtc(c)->pipe); + drm_framebuffer_unreference(c->primary->fb); + c->primary->fb = NULL; + } + } + mutex_unlock(&dev->struct_mutex); +} + +void intel_connector_unregister(struct intel_connector *intel_connector) +{ + struct drm_connector *connector = &intel_connector->base; + + intel_panel_destroy_backlight(connector); + drm_sysfs_connector_remove(connector); } void intel_modeset_cleanup(struct drm_device *dev) @@ -11343,9 +12304,9 @@ void intel_modeset_cleanup(struct drm_device *dev) intel_unregister_dsm_handler(); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + for_each_crtc(dev, crtc) { /* Skip inactive CRTCs */ - if (!crtc->fb) + if (!crtc->primary->fb) continue; intel_increase_pllclock(crtc); @@ -11364,13 +12325,19 @@ void intel_modeset_cleanup(struct drm_device *dev) /* destroy the backlight and sysfs files before encoders/connectors */ list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - intel_panel_destroy_backlight(connector); - drm_sysfs_connector_remove(connector); + struct intel_connector *intel_connector; + + intel_connector = to_intel_connector(connector); + intel_connector->unregister(intel_connector); } drm_mode_config_cleanup(dev); intel_cleanup_overlay(dev); + + mutex_lock(&dev->struct_mutex); + intel_cleanup_gt_powersave(dev); + mutex_unlock(&dev->struct_mutex); } /* @@ -11398,12 +12365,24 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state) unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL; u16 gmch_ctrl; - pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl); + if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) { + DRM_ERROR("failed to read control word\n"); + return -EIO; + } + + if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state) + return 0; + if (state) gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE; else gmch_ctrl |= INTEL_GMCH_VGA_DISABLE; - pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl); + + if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) { + DRM_ERROR("failed to write control word\n"); + return -EIO; + } + return 0; } @@ -11423,6 +12402,7 @@ struct intel_display_error_state { struct intel_pipe_error_state { bool power_domain_on; u32 source; + u32 stat; } pipe[I915_MAX_PIPES]; struct intel_plane_error_state { @@ -11453,7 +12433,7 @@ struct intel_display_error_state { struct intel_display_error_state * intel_display_capture_error_state(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_display_error_state *error; int transcoders[] = { TRANSCODER_A, @@ -11475,19 +12455,14 @@ intel_display_capture_error_state(struct drm_device *dev) for_each_pipe(i) { error->pipe[i].power_domain_on = - intel_display_power_enabled_sw(dev, POWER_DOMAIN_PIPE(i)); + intel_display_power_enabled_unlocked(dev_priv, + POWER_DOMAIN_PIPE(i)); if (!error->pipe[i].power_domain_on) continue; - if (INTEL_INFO(dev)->gen <= 6 || IS_VALLEYVIEW(dev)) { - error->cursor[i].control = I915_READ(CURCNTR(i)); - error->cursor[i].position = I915_READ(CURPOS(i)); - error->cursor[i].base = I915_READ(CURBASE(i)); - } else { - error->cursor[i].control = I915_READ(CURCNTR_IVB(i)); - error->cursor[i].position = I915_READ(CURPOS_IVB(i)); - error->cursor[i].base = I915_READ(CURBASE_IVB(i)); - } + error->cursor[i].control = I915_READ(CURCNTR(i)); + error->cursor[i].position = I915_READ(CURPOS(i)); + error->cursor[i].base = I915_READ(CURBASE(i)); error->plane[i].control = I915_READ(DSPCNTR(i)); error->plane[i].stride = I915_READ(DSPSTRIDE(i)); @@ -11503,6 +12478,9 @@ intel_display_capture_error_state(struct drm_device *dev) } error->pipe[i].source = I915_READ(PIPESRC(i)); + + if (!HAS_PCH_SPLIT(dev)) + error->pipe[i].stat = I915_READ(PIPESTAT(i)); } error->num_transcoders = INTEL_INFO(dev)->num_pipes; @@ -11513,7 +12491,7 @@ intel_display_capture_error_state(struct drm_device *dev) enum transcoder cpu_transcoder = transcoders[i]; error->transcoder[i].power_domain_on = - intel_display_power_enabled_sw(dev, + intel_display_power_enabled_unlocked(dev_priv, POWER_DOMAIN_TRANSCODER(cpu_transcoder)); if (!error->transcoder[i].power_domain_on) continue; @@ -11553,6 +12531,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, err_printf(m, " Power: %s\n", error->pipe[i].power_domain_on ? "on" : "off"); err_printf(m, " SRC: %08x\n", error->pipe[i].source); + err_printf(m, " STAT: %08x\n", error->pipe[i].stat); err_printf(m, "Plane [%d]:\n", i); err_printf(m, " CNTR: %08x\n", error->plane[i].control); diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 5ede4e8e290..8a1a4fbc06a 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -28,6 +28,8 @@ #include <linux/i2c.h> #include <linux/slab.h> #include <linux/export.h> +#include <linux/notifier.h> +#include <linux/reboot.h> #include <drm/drmP.h> #include <drm/drm_crtc.h> #include <drm/drm_crtc_helper.h> @@ -64,6 +66,24 @@ static const struct dp_link_dpll vlv_dpll[] = { { .p1 = 2, .p2 = 2, .n = 1, .m1 = 2, .m2 = 27 } } }; +/* + * CHV supports eDP 1.4 that have more link rates. + * Below only provides the fixed rate but exclude variable rate. + */ +static const struct dp_link_dpll chv_dpll[] = { + /* + * CHV requires to program fractional division for m2. + * m2 is stored in fixed point format using formula below + * (m2_int << 22) | m2_fraction + */ + { DP_LINK_BW_1_62, /* m2_int = 32, m2_fraction = 1677722 */ + { .p1 = 4, .p2 = 2, .n = 1, .m1 = 2, .m2 = 0x819999a } }, + { DP_LINK_BW_2_7, /* m2_int = 27, m2_fraction = 0 */ + { .p1 = 4, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } }, + { DP_LINK_BW_5_4, /* m2_int = 27, m2_fraction = 0 */ + { .p1 = 2, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } } +}; + /** * is_edp - is the given port attached to an eDP panel (either CPU or PCH) * @intel_dp: DP struct @@ -91,18 +111,26 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector) } static void intel_dp_link_down(struct intel_dp *intel_dp); +static bool _edp_panel_vdd_on(struct intel_dp *intel_dp); +static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync); static int intel_dp_max_link_bw(struct intel_dp *intel_dp) { int max_link_bw = intel_dp->dpcd[DP_MAX_LINK_RATE]; + struct drm_device *dev = intel_dp->attached_connector->base.dev; switch (max_link_bw) { case DP_LINK_BW_1_62: case DP_LINK_BW_2_7: break; case DP_LINK_BW_5_4: /* 1.2 capable displays may advertise higher bw */ - max_link_bw = DP_LINK_BW_2_7; + if (((IS_HASWELL(dev) && !IS_HSW_ULX(dev)) || + INTEL_INFO(dev)->gen >= 8) && + intel_dp->dpcd[DP_DPCD_REV] >= 0x12) + max_link_bw = DP_LINK_BW_5_4; + else + max_link_bw = DP_LINK_BW_2_7; break; default: WARN(1, "invalid max DP link bw val %x, using 1.62Gbps\n", @@ -113,6 +141,22 @@ intel_dp_max_link_bw(struct intel_dp *intel_dp) return max_link_bw; } +static u8 intel_dp_max_lane_count(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + u8 source_max, sink_max; + + source_max = 4; + if (HAS_DDI(dev) && intel_dig_port->port == PORT_A && + (intel_dig_port->saved_port_bits & DDI_A_4_LANES) == 0) + source_max = 2; + + sink_max = drm_dp_max_lane_count(intel_dp->dpcd); + + return min(source_max, sink_max); +} + /* * The units on the numbers in the next two are... bizarre. Examples will * make it clearer; this one parallels an example in the eDP spec. @@ -163,7 +207,7 @@ intel_dp_mode_valid(struct drm_connector *connector, } max_link_clock = drm_dp_bw_code_to_link_rate(intel_dp_max_link_bw(intel_dp)); - max_lanes = drm_dp_max_lane_count(intel_dp->dpcd); + max_lanes = intel_dp_max_lane_count(intel_dp); max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes); mode_rate = intel_dp_link_required(target_clock, 18); @@ -294,7 +338,38 @@ static u32 _pp_stat_reg(struct intel_dp *intel_dp) return VLV_PIPE_PP_STATUS(vlv_power_sequencer_pipe(intel_dp)); } -static bool ironlake_edp_have_panel_power(struct intel_dp *intel_dp) +/* Reboot notifier handler to shutdown panel power to guarantee T12 timing + This function only applicable when panel PM state is not to be tracked */ +static int edp_notify_handler(struct notifier_block *this, unsigned long code, + void *unused) +{ + struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp), + edp_notifier); + struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct drm_i915_private *dev_priv = dev->dev_private; + u32 pp_div; + u32 pp_ctrl_reg, pp_div_reg; + enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + + if (!is_edp(intel_dp) || code != SYS_RESTART) + return 0; + + if (IS_VALLEYVIEW(dev)) { + pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe); + pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe); + pp_div = I915_READ(pp_div_reg); + pp_div &= PP_REFERENCE_DIVIDER_MASK; + + /* 0x1F write to PP_DIV_REG sets max cycle delay */ + I915_WRITE(pp_div_reg, pp_div | 0x1F); + I915_WRITE(pp_ctrl_reg, PANEL_UNLOCK_REGS | PANEL_POWER_OFF); + msleep(intel_dp->panel_power_cycle_delay); + } + + return 0; +} + +static bool edp_have_panel_power(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; @@ -302,12 +377,17 @@ static bool ironlake_edp_have_panel_power(struct intel_dp *intel_dp) return (I915_READ(_pp_stat_reg(intel_dp)) & PP_ON) != 0; } -static bool ironlake_edp_have_panel_vdd(struct intel_dp *intel_dp) +static bool edp_have_panel_vdd(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; + enum intel_display_power_domain power_domain; - return (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0; + power_domain = intel_display_port_power_domain(intel_encoder); + return intel_display_power_enabled(dev_priv, power_domain) && + (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0; } static void @@ -319,7 +399,7 @@ intel_dp_check_edp(struct intel_dp *intel_dp) if (!is_edp(intel_dp)) return; - if (!ironlake_edp_have_panel_power(intel_dp) && !ironlake_edp_have_panel_vdd(intel_dp)) { + if (!edp_have_panel_power(intel_dp) && !edp_have_panel_vdd(intel_dp)) { WARN(1, "eDP powered off while attempting aux channel communication.\n"); DRM_DEBUG_KMS("Status 0x%08x Control 0x%08x\n", I915_READ(_pp_stat_reg(intel_dp)), @@ -351,31 +431,46 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq) return status; } -static uint32_t get_aux_clock_divider(struct intel_dp *intel_dp, - int index) +static uint32_t i9xx_get_aux_clock_divider(struct intel_dp *intel_dp, int index) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - /* The clock divider is based off the hrawclk, - * and would like to run at 2MHz. So, take the - * hrawclk value and divide by 2 and use that - * - * Note that PCH attached eDP panels should use a 125MHz input - * clock divider. + /* + * The clock divider is based off the hrawclk, and would like to run at + * 2MHz. So, take the hrawclk value and divide by 2 and use that */ - if (IS_VALLEYVIEW(dev)) { - return index ? 0 : 100; - } else if (intel_dig_port->port == PORT_A) { - if (index) - return 0; - if (HAS_DDI(dev)) - return DIV_ROUND_CLOSEST(intel_ddi_get_cdclk_freq(dev_priv), 2000); - else if (IS_GEN6(dev) || IS_GEN7(dev)) + return index ? 0 : intel_hrawclk(dev) / 2; +} + +static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + + if (index) + return 0; + + if (intel_dig_port->port == PORT_A) { + if (IS_GEN6(dev) || IS_GEN7(dev)) return 200; /* SNB & IVB eDP input clock at 400Mhz */ else return 225; /* eDP input clock at 450Mhz */ + } else { + return DIV_ROUND_UP(intel_pch_rawclk(dev), 2); + } +} + +static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + if (intel_dig_port->port == PORT_A) { + if (index) + return 0; + return DIV_ROUND_CLOSEST(intel_ddi_get_cdclk_freq(dev_priv), 2000); } else if (dev_priv->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) { /* Workaround for non-ULT HSW */ switch (index) { @@ -383,13 +478,46 @@ static uint32_t get_aux_clock_divider(struct intel_dp *intel_dp, case 1: return 72; default: return 0; } - } else if (HAS_PCH_SPLIT(dev)) { + } else { return index ? 0 : DIV_ROUND_UP(intel_pch_rawclk(dev), 2); - } else { - return index ? 0 :intel_hrawclk(dev) / 2; } } +static uint32_t vlv_get_aux_clock_divider(struct intel_dp *intel_dp, int index) +{ + return index ? 0 : 100; +} + +static uint32_t i9xx_get_aux_send_ctl(struct intel_dp *intel_dp, + bool has_aux_irq, + int send_bytes, + uint32_t aux_clock_divider) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + uint32_t precharge, timeout; + + if (IS_GEN6(dev)) + precharge = 3; + else + precharge = 5; + + if (IS_BROADWELL(dev) && intel_dp->aux_ch_ctl_reg == DPA_AUX_CH_CTL) + timeout = DP_AUX_CH_CTL_TIME_OUT_600us; + else + timeout = DP_AUX_CH_CTL_TIME_OUT_400us; + + return DP_AUX_CH_CTL_SEND_BUSY | + DP_AUX_CH_CTL_DONE | + (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) | + DP_AUX_CH_CTL_TIME_OUT_ERROR | + timeout | + DP_AUX_CH_CTL_RECEIVE_ERROR | + (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | + (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) | + (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT); +} + static int intel_dp_aux_ch(struct intel_dp *intel_dp, uint8_t *send, int send_bytes, @@ -403,9 +531,11 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, uint32_t aux_clock_divider; int i, ret, recv_bytes; uint32_t status; - int try, precharge, clock = 0; - bool has_aux_irq = true; - uint32_t timeout; + int try, clock = 0; + bool has_aux_irq = HAS_AUX_IRQ(dev); + bool vdd; + + vdd = _edp_panel_vdd_on(intel_dp); /* dp aux is extremely sensitive to irq latency, hence request the * lowest possible wakeup latency and so prevent the cpu from going into @@ -415,16 +545,6 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, intel_dp_check_edp(intel_dp); - if (IS_GEN6(dev)) - precharge = 3; - else - precharge = 5; - - if (IS_BROADWELL(dev) && ch_ctl == DPA_AUX_CH_CTL) - timeout = DP_AUX_CH_CTL_TIME_OUT_600us; - else - timeout = DP_AUX_CH_CTL_TIME_OUT_400us; - intel_aux_display_runtime_get(dev_priv); /* Try to wait for any previous AUX channel activity */ @@ -448,7 +568,12 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, goto out; } - while ((aux_clock_divider = get_aux_clock_divider(intel_dp, clock++))) { + while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) { + u32 send_ctl = intel_dp->get_aux_send_ctl(intel_dp, + has_aux_irq, + send_bytes, + aux_clock_divider); + /* Must try at least 3 times according to DP spec */ for (try = 0; try < 5; try++) { /* Load the send data into the aux channel data registers */ @@ -457,16 +582,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, pack_aux(send + i, send_bytes - i)); /* Send the command and wait for it to complete */ - I915_WRITE(ch_ctl, - DP_AUX_CH_CTL_SEND_BUSY | - (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) | - timeout | - (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | - (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) | - (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT) | - DP_AUX_CH_CTL_DONE | - DP_AUX_CH_CTL_TIME_OUT_ERROR | - DP_AUX_CH_CTL_RECEIVE_ERROR); + I915_WRITE(ch_ctl, send_ctl); status = intel_dp_aux_wait_done(intel_dp, has_aux_irq); @@ -525,239 +641,141 @@ out: pm_qos_update_request(&dev_priv->pm_qos, PM_QOS_DEFAULT_VALUE); intel_aux_display_runtime_put(dev_priv); + if (vdd) + edp_panel_vdd_off(intel_dp, false); + return ret; } -/* Write data to the aux channel in native mode */ -static int -intel_dp_aux_native_write(struct intel_dp *intel_dp, - uint16_t address, uint8_t *send, int send_bytes) +#define BARE_ADDRESS_SIZE 3 +#define HEADER_SIZE (BARE_ADDRESS_SIZE + 1) +static ssize_t +intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) { + struct intel_dp *intel_dp = container_of(aux, struct intel_dp, aux); + uint8_t txbuf[20], rxbuf[20]; + size_t txsize, rxsize; int ret; - uint8_t msg[20]; - int msg_bytes; - uint8_t ack; - if (WARN_ON(send_bytes > 16)) - return -E2BIG; + txbuf[0] = msg->request << 4; + txbuf[1] = msg->address >> 8; + txbuf[2] = msg->address & 0xff; + txbuf[3] = msg->size - 1; - intel_dp_check_edp(intel_dp); - msg[0] = DP_AUX_NATIVE_WRITE << 4; - msg[1] = address >> 8; - msg[2] = address & 0xff; - msg[3] = send_bytes - 1; - memcpy(&msg[4], send, send_bytes); - msg_bytes = send_bytes + 4; - for (;;) { - ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes, &ack, 1); - if (ret < 0) - return ret; - ack >>= 4; - if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_ACK) - break; - else if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_DEFER) - udelay(100); - else - return -EIO; - } - return send_bytes; -} + switch (msg->request & ~DP_AUX_I2C_MOT) { + case DP_AUX_NATIVE_WRITE: + case DP_AUX_I2C_WRITE: + txsize = msg->size ? HEADER_SIZE + msg->size : BARE_ADDRESS_SIZE; + rxsize = 1; -/* Write a single byte to the aux channel in native mode */ -static int -intel_dp_aux_native_write_1(struct intel_dp *intel_dp, - uint16_t address, uint8_t byte) -{ - return intel_dp_aux_native_write(intel_dp, address, &byte, 1); -} + if (WARN_ON(txsize > 20)) + return -E2BIG; -/* read bytes from a native aux channel */ -static int -intel_dp_aux_native_read(struct intel_dp *intel_dp, - uint16_t address, uint8_t *recv, int recv_bytes) -{ - uint8_t msg[4]; - int msg_bytes; - uint8_t reply[20]; - int reply_bytes; - uint8_t ack; - int ret; + memcpy(txbuf + HEADER_SIZE, msg->buffer, msg->size); - if (WARN_ON(recv_bytes > 19)) - return -E2BIG; + ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize); + if (ret > 0) { + msg->reply = rxbuf[0] >> 4; - intel_dp_check_edp(intel_dp); - msg[0] = DP_AUX_NATIVE_READ << 4; - msg[1] = address >> 8; - msg[2] = address & 0xff; - msg[3] = recv_bytes - 1; + /* Return payload size. */ + ret = msg->size; + } + break; - msg_bytes = 4; - reply_bytes = recv_bytes + 1; + case DP_AUX_NATIVE_READ: + case DP_AUX_I2C_READ: + txsize = msg->size ? HEADER_SIZE : BARE_ADDRESS_SIZE; + rxsize = msg->size + 1; - for (;;) { - ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes, - reply, reply_bytes); - if (ret == 0) - return -EPROTO; - if (ret < 0) - return ret; - ack = reply[0] >> 4; - if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_ACK) { - memcpy(recv, reply + 1, ret - 1); - return ret - 1; + if (WARN_ON(rxsize > 20)) + return -E2BIG; + + ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize); + if (ret > 0) { + msg->reply = rxbuf[0] >> 4; + /* + * Assume happy day, and copy the data. The caller is + * expected to check msg->reply before touching it. + * + * Return payload size. + */ + ret--; + memcpy(msg->buffer, rxbuf + 1, ret); } - else if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_DEFER) - udelay(100); - else - return -EIO; + break; + + default: + ret = -EINVAL; + break; } + + return ret; } -static int -intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode, - uint8_t write_byte, uint8_t *read_byte) -{ - struct i2c_algo_dp_aux_data *algo_data = adapter->algo_data; - struct intel_dp *intel_dp = container_of(adapter, - struct intel_dp, - adapter); - uint16_t address = algo_data->address; - uint8_t msg[5]; - uint8_t reply[2]; - unsigned retry; - int msg_bytes; - int reply_bytes; +static void +intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector) +{ + struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + enum port port = intel_dig_port->port; + const char *name = NULL; int ret; - ironlake_edp_panel_vdd_on(intel_dp); - intel_dp_check_edp(intel_dp); - /* Set up the command byte */ - if (mode & MODE_I2C_READ) - msg[0] = DP_AUX_I2C_READ << 4; - else - msg[0] = DP_AUX_I2C_WRITE << 4; - - if (!(mode & MODE_I2C_STOP)) - msg[0] |= DP_AUX_I2C_MOT << 4; - - msg[1] = address >> 8; - msg[2] = address; - - switch (mode) { - case MODE_I2C_WRITE: - msg[3] = 0; - msg[4] = write_byte; - msg_bytes = 5; - reply_bytes = 1; + switch (port) { + case PORT_A: + intel_dp->aux_ch_ctl_reg = DPA_AUX_CH_CTL; + name = "DPDDC-A"; break; - case MODE_I2C_READ: - msg[3] = 0; - msg_bytes = 4; - reply_bytes = 2; + case PORT_B: + intel_dp->aux_ch_ctl_reg = PCH_DPB_AUX_CH_CTL; + name = "DPDDC-B"; break; - default: - msg_bytes = 3; - reply_bytes = 1; + case PORT_C: + intel_dp->aux_ch_ctl_reg = PCH_DPC_AUX_CH_CTL; + name = "DPDDC-C"; + break; + case PORT_D: + intel_dp->aux_ch_ctl_reg = PCH_DPD_AUX_CH_CTL; + name = "DPDDC-D"; break; + default: + BUG(); } - /* - * DP1.2 sections 2.7.7.1.5.6.1 and 2.7.7.1.6.6.1: A DP Source device is - * required to retry at least seven times upon receiving AUX_DEFER - * before giving up the AUX transaction. - */ - for (retry = 0; retry < 7; retry++) { - ret = intel_dp_aux_ch(intel_dp, - msg, msg_bytes, - reply, reply_bytes); - if (ret < 0) { - DRM_DEBUG_KMS("aux_ch failed %d\n", ret); - goto out; - } + if (!HAS_DDI(dev)) + intel_dp->aux_ch_ctl_reg = intel_dp->output_reg + 0x10; - switch ((reply[0] >> 4) & DP_AUX_NATIVE_REPLY_MASK) { - case DP_AUX_NATIVE_REPLY_ACK: - /* I2C-over-AUX Reply field is only valid - * when paired with AUX ACK. - */ - break; - case DP_AUX_NATIVE_REPLY_NACK: - DRM_DEBUG_KMS("aux_ch native nack\n"); - ret = -EREMOTEIO; - goto out; - case DP_AUX_NATIVE_REPLY_DEFER: - /* - * For now, just give more slack to branch devices. We - * could check the DPCD for I2C bit rate capabilities, - * and if available, adjust the interval. We could also - * be more careful with DP-to-Legacy adapters where a - * long legacy cable may force very low I2C bit rates. - */ - if (intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & - DP_DWN_STRM_PORT_PRESENT) - usleep_range(500, 600); - else - usleep_range(300, 400); - continue; - default: - DRM_ERROR("aux_ch invalid native reply 0x%02x\n", - reply[0]); - ret = -EREMOTEIO; - goto out; - } + intel_dp->aux.name = name; + intel_dp->aux.dev = dev->dev; + intel_dp->aux.transfer = intel_dp_aux_transfer; - switch ((reply[0] >> 4) & DP_AUX_I2C_REPLY_MASK) { - case DP_AUX_I2C_REPLY_ACK: - if (mode == MODE_I2C_READ) { - *read_byte = reply[1]; - } - ret = reply_bytes - 1; - goto out; - case DP_AUX_I2C_REPLY_NACK: - DRM_DEBUG_KMS("aux_i2c nack\n"); - ret = -EREMOTEIO; - goto out; - case DP_AUX_I2C_REPLY_DEFER: - DRM_DEBUG_KMS("aux_i2c defer\n"); - udelay(100); - break; - default: - DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]); - ret = -EREMOTEIO; - goto out; - } - } + DRM_DEBUG_KMS("registering %s bus for %s\n", name, + connector->base.kdev->kobj.name); - DRM_ERROR("too many retries, giving up\n"); - ret = -EREMOTEIO; + ret = drm_dp_aux_register(&intel_dp->aux); + if (ret < 0) { + DRM_ERROR("drm_dp_aux_register() for %s failed (%d)\n", + name, ret); + return; + } -out: - ironlake_edp_panel_vdd_off(intel_dp, false); - return ret; + ret = sysfs_create_link(&connector->base.kdev->kobj, + &intel_dp->aux.ddc.dev.kobj, + intel_dp->aux.ddc.dev.kobj.name); + if (ret < 0) { + DRM_ERROR("sysfs_create_link() for %s failed (%d)\n", name, ret); + drm_dp_aux_unregister(&intel_dp->aux); + } } -static int -intel_dp_i2c_init(struct intel_dp *intel_dp, - struct intel_connector *intel_connector, const char *name) +static void +intel_dp_connector_unregister(struct intel_connector *intel_connector) { - int ret; - - DRM_DEBUG_KMS("i2c_init %s\n", name); - intel_dp->algo.running = false; - intel_dp->algo.address = 0; - intel_dp->algo.aux_ch = intel_dp_i2c_aux_ch; + struct intel_dp *intel_dp = intel_attached_dp(&intel_connector->base); - memset(&intel_dp->adapter, '\0', sizeof(intel_dp->adapter)); - intel_dp->adapter.owner = THIS_MODULE; - intel_dp->adapter.class = I2C_CLASS_DDC; - strncpy(intel_dp->adapter.name, name, sizeof(intel_dp->adapter.name) - 1); - intel_dp->adapter.name[sizeof(intel_dp->adapter.name) - 1] = '\0'; - intel_dp->adapter.algo_data = &intel_dp->algo; - intel_dp->adapter.dev.parent = intel_connector->base.kdev; - - ret = i2c_dp_aux_add_bus(&intel_dp->adapter); - return ret; + sysfs_remove_link(&intel_connector->base.kdev->kobj, + intel_dp->aux.ddc.dev.kobj.name); + intel_connector_unregister(intel_connector); } static void @@ -776,6 +794,9 @@ intel_dp_set_clock(struct intel_encoder *encoder, } else if (HAS_PCH_SPLIT(dev)) { divisor = pch_dpll; count = ARRAY_SIZE(pch_dpll); + } else if (IS_CHERRYVIEW(dev)) { + divisor = chv_dpll; + count = ARRAY_SIZE(chv_dpll); } else if (IS_VALLEYVIEW(dev)) { divisor = vlv_dpll; count = ARRAY_SIZE(vlv_dpll); @@ -792,6 +813,20 @@ intel_dp_set_clock(struct intel_encoder *encoder, } } +static void +intel_dp_set_m2_n2(struct intel_crtc *crtc, struct intel_link_m_n *m_n) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum transcoder transcoder = crtc->config.cpu_transcoder; + + I915_WRITE(PIPE_DATA_M2(transcoder), + TU_SIZE(m_n->tu) | m_n->gmch_m); + I915_WRITE(PIPE_DATA_N2(transcoder), m_n->gmch_n); + I915_WRITE(PIPE_LINK_M2(transcoder), m_n->link_m); + I915_WRITE(PIPE_LINK_N2(transcoder), m_n->link_n); +} + bool intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_config *pipe_config) @@ -804,16 +839,20 @@ intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc *intel_crtc = encoder->new_crtc; struct intel_connector *intel_connector = intel_dp->attached_connector; int lane_count, clock; - int max_lane_count = drm_dp_max_lane_count(intel_dp->dpcd); - int max_clock = intel_dp_max_link_bw(intel_dp) == DP_LINK_BW_2_7 ? 1 : 0; + int min_lane_count = 1; + int max_lane_count = intel_dp_max_lane_count(intel_dp); + /* Conveniently, the link BW constants become indices with a shift...*/ + int min_clock = 0; + int max_clock = intel_dp_max_link_bw(intel_dp) >> 3; int bpp, mode_rate; - static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 }; + static int bws[] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7, DP_LINK_BW_5_4 }; int link_avail, link_clock; if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A) pipe_config->has_pch_encoder = true; pipe_config->has_dp_encoder = true; + pipe_config->has_audio = intel_dp->has_audio; if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) { intel_fixed_panel_mode(intel_connector->panel.fixed_mode, @@ -837,19 +876,38 @@ intel_dp_compute_config(struct intel_encoder *encoder, /* Walk through all bpp values. Luckily they're all nicely spaced with 2 * bpc in between. */ bpp = pipe_config->pipe_bpp; - if (is_edp(intel_dp) && dev_priv->vbt.edp_bpp && - dev_priv->vbt.edp_bpp < bpp) { - DRM_DEBUG_KMS("clamping bpp for eDP panel to BIOS-provided %i\n", - dev_priv->vbt.edp_bpp); - bpp = dev_priv->vbt.edp_bpp; + if (is_edp(intel_dp)) { + if (dev_priv->vbt.edp_bpp && dev_priv->vbt.edp_bpp < bpp) { + DRM_DEBUG_KMS("clamping bpp for eDP panel to BIOS-provided %i\n", + dev_priv->vbt.edp_bpp); + bpp = dev_priv->vbt.edp_bpp; + } + + if (IS_BROADWELL(dev)) { + /* Yes, it's an ugly hack. */ + min_lane_count = max_lane_count; + DRM_DEBUG_KMS("forcing lane count to max (%u) on BDW\n", + min_lane_count); + } else if (dev_priv->vbt.edp_lanes) { + min_lane_count = min(dev_priv->vbt.edp_lanes, + max_lane_count); + DRM_DEBUG_KMS("using min %u lanes per VBT\n", + min_lane_count); + } + + if (dev_priv->vbt.edp_rate) { + min_clock = min(dev_priv->vbt.edp_rate >> 3, max_clock); + DRM_DEBUG_KMS("using min %02x link bw per VBT\n", + bws[min_clock]); + } } for (; bpp >= 6*3; bpp -= 2*3) { mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock, bpp); - for (clock = 0; clock <= max_clock; clock++) { - for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) { + for (clock = min_clock; clock <= max_clock; clock++) { + for (lane_count = min_lane_count; lane_count <= max_lane_count; lane_count <<= 1) { link_clock = drm_dp_bw_code_to_link_rate(bws[clock]); link_avail = intel_dp_max_data_rate(link_clock, lane_count); @@ -895,6 +953,14 @@ found: pipe_config->port_clock, &pipe_config->dp_m_n); + if (intel_connector->panel.downclock_mode != NULL && + intel_dp->drrs_state.type == SEAMLESS_DRRS_SUPPORT) { + intel_link_compute_m_n(bpp, lane_count, + intel_connector->panel.downclock_mode->clock, + pipe_config->port_clock, + &pipe_config->dp_m2_n2); + } + intel_dp_set_clock(encoder, pipe_config, intel_dp->link_bw); return true; @@ -930,7 +996,7 @@ static void ironlake_set_pll_cpu_edp(struct intel_dp *intel_dp) udelay(500); } -static void intel_dp_mode_set(struct intel_encoder *encoder) +static void intel_dp_prepare(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -965,7 +1031,7 @@ static void intel_dp_mode_set(struct intel_encoder *encoder) intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0; intel_dp->DP |= DP_PORT_WIDTH(intel_dp->lane_count); - if (intel_dp->has_audio) { + if (crtc->config.has_audio) { DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n", pipe_name(crtc->pipe)); intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE; @@ -998,26 +1064,27 @@ static void intel_dp_mode_set(struct intel_encoder *encoder) if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) intel_dp->DP |= DP_ENHANCED_FRAMING; - if (crtc->pipe == 1) - intel_dp->DP |= DP_PIPEB_SELECT; + if (!IS_CHERRYVIEW(dev)) { + if (crtc->pipe == 1) + intel_dp->DP |= DP_PIPEB_SELECT; + } else { + intel_dp->DP |= DP_PIPE_SELECT_CHV(crtc->pipe); + } } else { intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; } - - if (port == PORT_A && !IS_VALLEYVIEW(dev)) - ironlake_set_pll_cpu_edp(intel_dp); } -#define IDLE_ON_MASK (PP_ON | 0 | PP_SEQUENCE_MASK | 0 | PP_SEQUENCE_STATE_MASK) -#define IDLE_ON_VALUE (PP_ON | 0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_ON_IDLE) +#define IDLE_ON_MASK (PP_ON | PP_SEQUENCE_MASK | 0 | PP_SEQUENCE_STATE_MASK) +#define IDLE_ON_VALUE (PP_ON | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_ON_IDLE) -#define IDLE_OFF_MASK (PP_ON | 0 | PP_SEQUENCE_MASK | 0 | PP_SEQUENCE_STATE_MASK) -#define IDLE_OFF_VALUE (0 | 0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) +#define IDLE_OFF_MASK (PP_ON | PP_SEQUENCE_MASK | 0 | 0) +#define IDLE_OFF_VALUE (0 | PP_SEQUENCE_NONE | 0 | 0) -#define IDLE_CYCLE_MASK (PP_ON | 0 | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK) -#define IDLE_CYCLE_VALUE (0 | 0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) +#define IDLE_CYCLE_MASK (PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK) +#define IDLE_CYCLE_VALUE (0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) -static void ironlake_wait_panel_status(struct intel_dp *intel_dp, +static void wait_panel_status(struct intel_dp *intel_dp, u32 mask, u32 value) { @@ -1042,24 +1109,41 @@ static void ironlake_wait_panel_status(struct intel_dp *intel_dp, DRM_DEBUG_KMS("Wait complete\n"); } -static void ironlake_wait_panel_on(struct intel_dp *intel_dp) +static void wait_panel_on(struct intel_dp *intel_dp) { DRM_DEBUG_KMS("Wait for panel power on\n"); - ironlake_wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE); + wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE); } -static void ironlake_wait_panel_off(struct intel_dp *intel_dp) +static void wait_panel_off(struct intel_dp *intel_dp) { DRM_DEBUG_KMS("Wait for panel power off time\n"); - ironlake_wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE); + wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE); } -static void ironlake_wait_panel_power_cycle(struct intel_dp *intel_dp) +static void wait_panel_power_cycle(struct intel_dp *intel_dp) { DRM_DEBUG_KMS("Wait for panel power cycle\n"); - ironlake_wait_panel_status(intel_dp, IDLE_CYCLE_MASK, IDLE_CYCLE_VALUE); + + /* When we disable the VDD override bit last we have to do the manual + * wait. */ + wait_remaining_ms_from_jiffies(intel_dp->last_power_cycle, + intel_dp->panel_power_cycle_delay); + + wait_panel_status(intel_dp, IDLE_CYCLE_MASK, IDLE_CYCLE_VALUE); +} + +static void wait_backlight_on(struct intel_dp *intel_dp) +{ + wait_remaining_ms_from_jiffies(intel_dp->last_power_on, + intel_dp->backlight_on_delay); } +static void edp_wait_backlight_off(struct intel_dp *intel_dp) +{ + wait_remaining_ms_from_jiffies(intel_dp->last_backlight_off, + intel_dp->backlight_off_delay); +} /* Read the current pp_control value, unlocking the register if it * is locked @@ -1077,30 +1161,32 @@ static u32 ironlake_get_pp_control(struct intel_dp *intel_dp) return control; } -void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp) +static bool _edp_panel_vdd_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_i915_private *dev_priv = dev->dev_private; + enum intel_display_power_domain power_domain; u32 pp; u32 pp_stat_reg, pp_ctrl_reg; + bool need_to_disable = !intel_dp->want_panel_vdd; if (!is_edp(intel_dp)) - return; - - WARN(intel_dp->want_panel_vdd, - "eDP VDD already requested on\n"); + return false; intel_dp->want_panel_vdd = true; - if (ironlake_edp_have_panel_vdd(intel_dp)) - return; + if (edp_have_panel_vdd(intel_dp)) + return need_to_disable; - intel_runtime_pm_get(dev_priv); + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); DRM_DEBUG_KMS("Turning eDP VDD on\n"); - if (!ironlake_edp_have_panel_power(intel_dp)) - ironlake_wait_panel_power_cycle(intel_dp); + if (!edp_have_panel_power(intel_dp)) + wait_panel_power_cycle(intel_dp); pp = ironlake_get_pp_control(intel_dp); pp |= EDP_FORCE_VDD; @@ -1115,22 +1201,38 @@ void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp) /* * If the panel wasn't on, delay before accessing aux channel */ - if (!ironlake_edp_have_panel_power(intel_dp)) { + if (!edp_have_panel_power(intel_dp)) { DRM_DEBUG_KMS("eDP was not running\n"); msleep(intel_dp->panel_power_up_delay); } + + return need_to_disable; } -static void ironlake_panel_vdd_off_sync(struct intel_dp *intel_dp) +void intel_edp_panel_vdd_on(struct intel_dp *intel_dp) +{ + if (is_edp(intel_dp)) { + bool vdd = _edp_panel_vdd_on(intel_dp); + + WARN(!vdd, "eDP VDD already requested on\n"); + } +} + +static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; u32 pp_stat_reg, pp_ctrl_reg; - WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); + WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + + if (!intel_dp->want_panel_vdd && edp_have_panel_vdd(intel_dp)) { + struct intel_digital_port *intel_dig_port = + dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; + enum intel_display_power_domain power_domain; - if (!intel_dp->want_panel_vdd && ironlake_edp_have_panel_vdd(intel_dp)) { DRM_DEBUG_KMS("Turning eDP VDD off\n"); pp = ironlake_get_pp_control(intel_dp); @@ -1147,24 +1249,25 @@ static void ironlake_panel_vdd_off_sync(struct intel_dp *intel_dp) I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); if ((pp & POWER_TARGET_ON) == 0) - msleep(intel_dp->panel_power_cycle_delay); + intel_dp->last_power_cycle = jiffies; - intel_runtime_pm_put(dev_priv); + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_put(dev_priv, power_domain); } } -static void ironlake_panel_vdd_work(struct work_struct *__work) +static void edp_panel_vdd_work(struct work_struct *__work) { struct intel_dp *intel_dp = container_of(to_delayed_work(__work), struct intel_dp, panel_vdd_work); struct drm_device *dev = intel_dp_to_dev(intel_dp); - mutex_lock(&dev->mode_config.mutex); - ironlake_panel_vdd_off_sync(intel_dp); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + edp_panel_vdd_off_sync(intel_dp); + drm_modeset_unlock(&dev->mode_config.connection_mutex); } -void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) +static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) { if (!is_edp(intel_dp)) return; @@ -1174,7 +1277,7 @@ void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) intel_dp->want_panel_vdd = false; if (sync) { - ironlake_panel_vdd_off_sync(intel_dp); + edp_panel_vdd_off_sync(intel_dp); } else { /* * Queue the timer to fire a long @@ -1186,7 +1289,7 @@ void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) } } -void ironlake_edp_panel_on(struct intel_dp *intel_dp) +void intel_edp_panel_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; @@ -1198,12 +1301,12 @@ void ironlake_edp_panel_on(struct intel_dp *intel_dp) DRM_DEBUG_KMS("Turn eDP power on\n"); - if (ironlake_edp_have_panel_power(intel_dp)) { + if (edp_have_panel_power(intel_dp)) { DRM_DEBUG_KMS("eDP power already on\n"); return; } - ironlake_wait_panel_power_cycle(intel_dp); + wait_panel_power_cycle(intel_dp); pp_ctrl_reg = _pp_ctrl_reg(intel_dp); pp = ironlake_get_pp_control(intel_dp); @@ -1221,7 +1324,8 @@ void ironlake_edp_panel_on(struct intel_dp *intel_dp) I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); - ironlake_wait_panel_on(intel_dp); + wait_panel_on(intel_dp); + intel_dp->last_power_on = jiffies; if (IS_GEN5(dev)) { pp |= PANEL_POWER_RESET; /* restore panel reset bit */ @@ -1230,10 +1334,13 @@ void ironlake_edp_panel_on(struct intel_dp *intel_dp) } } -void ironlake_edp_panel_off(struct intel_dp *intel_dp) +void intel_edp_panel_off(struct intel_dp *intel_dp) { + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; + enum intel_display_power_domain power_domain; u32 pp; u32 pp_ctrl_reg; @@ -1242,20 +1349,32 @@ void ironlake_edp_panel_off(struct intel_dp *intel_dp) DRM_DEBUG_KMS("Turn eDP power off\n"); + edp_wait_backlight_off(intel_dp); + + WARN(!intel_dp->want_panel_vdd, "Need VDD to turn off panel\n"); + pp = ironlake_get_pp_control(intel_dp); /* We need to switch off panel power _and_ force vdd, for otherwise some * panels get very unhappy and cease to work. */ - pp &= ~(POWER_TARGET_ON | PANEL_POWER_RESET | EDP_BLC_ENABLE); + pp &= ~(POWER_TARGET_ON | PANEL_POWER_RESET | EDP_FORCE_VDD | + EDP_BLC_ENABLE); pp_ctrl_reg = _pp_ctrl_reg(intel_dp); + intel_dp->want_panel_vdd = false; + I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); - ironlake_wait_panel_off(intel_dp); + intel_dp->last_power_cycle = jiffies; + wait_panel_off(intel_dp); + + /* We got a reference when we enabled the VDD. */ + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_put(dev_priv, power_domain); } -void ironlake_edp_backlight_on(struct intel_dp *intel_dp) +void intel_edp_backlight_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; @@ -1273,7 +1392,7 @@ void ironlake_edp_backlight_on(struct intel_dp *intel_dp) * link. So delay a bit to make sure the image is solid before * allowing it to appear. */ - msleep(intel_dp->backlight_on_delay); + wait_backlight_on(intel_dp); pp = ironlake_get_pp_control(intel_dp); pp |= EDP_BLC_ENABLE; @@ -1285,7 +1404,7 @@ void ironlake_edp_backlight_on(struct intel_dp *intel_dp) intel_panel_enable_backlight(intel_dp->attached_connector); } -void ironlake_edp_backlight_off(struct intel_dp *intel_dp) +void intel_edp_backlight_off(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; @@ -1305,7 +1424,7 @@ void ironlake_edp_backlight_off(struct intel_dp *intel_dp) I915_WRITE(pp_ctrl_reg, pp); POSTING_READ(pp_ctrl_reg); - msleep(intel_dp->backlight_off_delay); + intel_dp->last_backlight_off = jiffies; } static void ironlake_edp_pll_on(struct intel_dp *intel_dp) @@ -1369,8 +1488,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) return; if (mode != DRM_MODE_DPMS_ON) { - ret = intel_dp_aux_native_write_1(intel_dp, DP_SET_POWER, - DP_SET_POWER_D3); + ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, + DP_SET_POWER_D3); if (ret != 1) DRM_DEBUG_DRIVER("failed to write sink power state\n"); } else { @@ -1379,9 +1498,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) * time to wake up. */ for (i = 0; i < 3; i++) { - ret = intel_dp_aux_native_write_1(intel_dp, - DP_SET_POWER, - DP_SET_POWER_D0); + ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, + DP_SET_POWER_D0); if (ret == 1) break; msleep(1); @@ -1396,13 +1514,22 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder, enum port port = dp_to_dig_port(intel_dp)->port; struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u32 tmp = I915_READ(intel_dp->output_reg); + enum intel_display_power_domain power_domain; + u32 tmp; + + power_domain = intel_display_port_power_domain(encoder); + if (!intel_display_power_enabled(dev_priv, power_domain)) + return false; + + tmp = I915_READ(intel_dp->output_reg); if (!(tmp & DP_PORT_EN)) return false; if (port == PORT_A && IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) { *pipe = PORT_TO_PIPE_CPT(tmp); + } else if (IS_CHERRYVIEW(dev)) { + *pipe = DP_PORT_TO_PIPE_CHV(tmp); } else if (!HAS_PCH_CPT(dev) || port == PORT_A) { *pipe = PORT_TO_PIPE(tmp); } else { @@ -1450,8 +1577,11 @@ static void intel_dp_get_config(struct intel_encoder *encoder, struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); int dotclock; + tmp = I915_READ(intel_dp->output_reg); + if (tmp & DP_AUDIO_OUTPUT_ENABLE) + pipe_config->has_audio = true; + if ((port == PORT_A) || !HAS_PCH_CPT(dev)) { - tmp = I915_READ(intel_dp->output_reg); if (tmp & DP_SYNC_HS_HIGH) flags |= DRM_MODE_FLAG_PHSYNC; else @@ -1590,19 +1720,19 @@ static void intel_edp_psr_enable_sink(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t aux_clock_divider = get_aux_clock_divider(intel_dp, 0); + uint32_t aux_clock_divider; int precharge = 0x3; int msg_size = 5; /* Header(4) + Message(1) */ + aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0); + /* Enable PSR in sink */ if (intel_dp->psr_dpcd[1] & DP_PSR_NO_TRAIN_ON_EXIT) - intel_dp_aux_native_write_1(intel_dp, DP_PSR_EN_CFG, - DP_PSR_ENABLE & - ~DP_PSR_MAIN_LINK_ACTIVE); + drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, + DP_PSR_ENABLE & ~DP_PSR_MAIN_LINK_ACTIVE); else - intel_dp_aux_native_write_1(intel_dp, DP_PSR_EN_CFG, - DP_PSR_ENABLE | - DP_PSR_MAIN_LINK_ACTIVE); + drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, + DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE); /* Setup AUX registers */ I915_WRITE(EDP_PSR_AUX_DATA1(dev), EDP_PSR_DPCD_COMMAND); @@ -1632,7 +1762,7 @@ static void intel_edp_psr_enable_source(struct intel_dp *intel_dp) val |= EDP_PSR_LINK_DISABLE; I915_WRITE(EDP_PSR_CTL(dev), val | - IS_BROADWELL(dev) ? 0 : link_entry_time | + (IS_BROADWELL(dev) ? 0 : link_entry_time) | max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT | idle_frames << EDP_PSR_IDLE_FRAME_SHIFT | EDP_PSR_ENABLE); @@ -1645,7 +1775,7 @@ static bool intel_edp_psr_match_conditions(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dev->dev_private; struct drm_crtc *crtc = dig_port->base.base.crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_i915_gem_object *obj = to_intel_framebuffer(crtc->fb)->obj; + struct drm_i915_gem_object *obj = to_intel_framebuffer(crtc->primary->fb)->obj; struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base; dev_priv->psr.source_ok = false; @@ -1661,7 +1791,7 @@ static bool intel_edp_psr_match_conditions(struct intel_dp *intel_dp) return false; } - if (!i915_enable_psr) { + if (!i915.enable_psr) { DRM_DEBUG_KMS("PSR disable by flag\n"); return false; } @@ -1678,7 +1808,7 @@ static bool intel_edp_psr_match_conditions(struct intel_dp *intel_dp) return false; } - obj = to_intel_framebuffer(crtc->fb)->obj; + obj = to_intel_framebuffer(crtc->primary->fb)->obj; if (obj->tiling_mode != I915_TILING_X || obj->fence_reg == I915_FENCE_REG_NONE) { DRM_DEBUG_KMS("PSR condition failed: fb not tiled or fenced\n"); @@ -1777,26 +1907,69 @@ static void intel_disable_dp(struct intel_encoder *encoder) /* Make sure the panel is off before trying to change the mode. But also * ensure that we have vdd while we switch off the panel. */ - ironlake_edp_backlight_off(intel_dp); + intel_edp_panel_vdd_on(intel_dp); + intel_edp_backlight_off(intel_dp); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); - ironlake_edp_panel_off(intel_dp); + intel_edp_panel_off(intel_dp); /* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */ if (!(port == PORT_A || IS_VALLEYVIEW(dev))) intel_dp_link_down(intel_dp); } -static void intel_post_disable_dp(struct intel_encoder *encoder) +static void g4x_post_disable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; + + if (port != PORT_A) + return; + + intel_dp_link_down(intel_dp); + ironlake_edp_pll_off(intel_dp); +} + +static void vlv_post_disable_dp(struct intel_encoder *encoder) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + + intel_dp_link_down(intel_dp); +} + +static void chv_post_disable_dp(struct intel_encoder *encoder) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_digital_port *dport = dp_to_dig_port(intel_dp); struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + enum pipe pipe = intel_crtc->pipe; + u32 val; - if (port == PORT_A || IS_VALLEYVIEW(dev)) { - intel_dp_link_down(intel_dp); - if (!IS_VALLEYVIEW(dev)) - ironlake_edp_pll_off(intel_dp); - } + intel_dp_link_down(intel_dp); + + mutex_lock(&dev_priv->dpio_lock); + + /* Propagate soft reset to data lane reset */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + + mutex_unlock(&dev_priv->dpio_lock); } static void intel_enable_dp(struct intel_encoder *encoder) @@ -1809,11 +1982,11 @@ static void intel_enable_dp(struct intel_encoder *encoder) if (WARN_ON(dp_reg & DP_PORT_EN)) return; - ironlake_edp_panel_vdd_on(intel_dp); + intel_edp_panel_vdd_on(intel_dp); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); intel_dp_start_link_train(intel_dp); - ironlake_edp_panel_on(intel_dp); - ironlake_edp_panel_vdd_off(intel_dp, true); + intel_edp_panel_on(intel_dp); + edp_panel_vdd_off(intel_dp, true); intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); } @@ -1823,14 +1996,14 @@ static void g4x_enable_dp(struct intel_encoder *encoder) struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); intel_enable_dp(encoder); - ironlake_edp_backlight_on(intel_dp); + intel_edp_backlight_on(intel_dp); } static void vlv_enable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - ironlake_edp_backlight_on(intel_dp); + intel_edp_backlight_on(intel_dp); } static void g4x_pre_enable_dp(struct intel_encoder *encoder) @@ -1838,8 +2011,13 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder) struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - if (dport->port == PORT_A) + intel_dp_prepare(encoder); + + /* Only ilk+ has port A */ + if (dport->port == PORT_A) { + ironlake_set_pll_cpu_edp(intel_dp); ironlake_edp_pll_on(intel_dp); + } } static void vlv_pre_enable_dp(struct intel_encoder *encoder) @@ -1869,10 +2047,12 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder) mutex_unlock(&dev_priv->dpio_lock); - /* init power sequencer on this pipe and port */ - intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, - &power_seq); + if (is_edp(intel_dp)) { + /* init power sequencer on this pipe and port */ + intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, + &power_seq); + } intel_enable_dp(encoder); @@ -1889,6 +2069,8 @@ static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder) enum dpio_channel port = vlv_dport_to_channel(dport); int pipe = intel_crtc->pipe; + intel_dp_prepare(encoder); + /* Program Tx lane resets to default */ mutex_lock(&dev_priv->dpio_lock); vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), @@ -1907,29 +2089,91 @@ static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder) mutex_unlock(&dev_priv->dpio_lock); } +static void chv_pre_enable_dp(struct intel_encoder *encoder) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_digital_port *dport = dp_to_dig_port(intel_dp); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct edp_power_seq power_seq; + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + int data, i; + u32 val; + + mutex_lock(&dev_priv->dpio_lock); + + /* Deassert soft data lane reset*/ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + + /* Program Tx lane latency optimal setting*/ + for (i = 0; i < 4; i++) { + /* Set the latency optimal bit */ + data = (i == 1) ? 0x0 : 0x6; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW11(ch, i), + data << DPIO_FRC_LATENCY_SHFIT); + + /* Set the upar bit */ + data = (i == 1) ? 0x0 : 0x1; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), + data << DPIO_UPAR_SHIFT); + } + + /* Data lane stagger programming */ + /* FIXME: Fix up value only after power analysis */ + + mutex_unlock(&dev_priv->dpio_lock); + + if (is_edp(intel_dp)) { + /* init power sequencer on this pipe and port */ + intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, + &power_seq); + } + + intel_enable_dp(encoder); + + vlv_wait_port_ready(dev_priv, dport); +} + /* * Native read with retry for link status and receiver capability reads for * cases where the sink may still be asleep. + * + * Sinks are *supposed* to come up within 1ms from an off state, but we're also + * supposed to retry 3 times per the spec. */ -static bool -intel_dp_aux_native_read_retry(struct intel_dp *intel_dp, uint16_t address, - uint8_t *recv, int recv_bytes) +static ssize_t +intel_dp_dpcd_read_wake(struct drm_dp_aux *aux, unsigned int offset, + void *buffer, size_t size) { - int ret, i; + ssize_t ret; + int i; - /* - * Sinks are *supposed* to come up within 1ms from an off state, - * but we're also supposed to retry 3 times per the spec. - */ for (i = 0; i < 3; i++) { - ret = intel_dp_aux_native_read(intel_dp, address, recv, - recv_bytes); - if (ret == recv_bytes) - return true; + ret = drm_dp_dpcd_read(aux, offset, buffer, size); + if (ret == size) + return ret; msleep(1); } - return false; + return ret; } /* @@ -1939,10 +2183,10 @@ intel_dp_aux_native_read_retry(struct intel_dp *intel_dp, uint16_t address, static bool intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]) { - return intel_dp_aux_native_read_retry(intel_dp, - DP_LANE0_1_STATUS, - link_status, - DP_LINK_STATUS_SIZE); + return intel_dp_dpcd_read_wake(&intel_dp->aux, + DP_LANE0_1_STATUS, + link_status, + DP_LINK_STATUS_SIZE) == DP_LINK_STATUS_SIZE; } /* @@ -2132,6 +2376,166 @@ static uint32_t intel_vlv_signal_levels(struct intel_dp *intel_dp) return 0; } +static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp) +{ + struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_digital_port *dport = dp_to_dig_port(intel_dp); + struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc); + u32 deemph_reg_value, margin_reg_value, val; + uint8_t train_set = intel_dp->train_set[0]; + enum dpio_channel ch = vlv_dport_to_channel(dport); + enum pipe pipe = intel_crtc->pipe; + int i; + + switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { + case DP_TRAIN_PRE_EMPHASIS_0: + switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { + case DP_TRAIN_VOLTAGE_SWING_400: + deemph_reg_value = 128; + margin_reg_value = 52; + break; + case DP_TRAIN_VOLTAGE_SWING_600: + deemph_reg_value = 128; + margin_reg_value = 77; + break; + case DP_TRAIN_VOLTAGE_SWING_800: + deemph_reg_value = 128; + margin_reg_value = 102; + break; + case DP_TRAIN_VOLTAGE_SWING_1200: + deemph_reg_value = 128; + margin_reg_value = 154; + /* FIXME extra to set for 1200 */ + break; + default: + return 0; + } + break; + case DP_TRAIN_PRE_EMPHASIS_3_5: + switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { + case DP_TRAIN_VOLTAGE_SWING_400: + deemph_reg_value = 85; + margin_reg_value = 78; + break; + case DP_TRAIN_VOLTAGE_SWING_600: + deemph_reg_value = 85; + margin_reg_value = 116; + break; + case DP_TRAIN_VOLTAGE_SWING_800: + deemph_reg_value = 85; + margin_reg_value = 154; + break; + default: + return 0; + } + break; + case DP_TRAIN_PRE_EMPHASIS_6: + switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { + case DP_TRAIN_VOLTAGE_SWING_400: + deemph_reg_value = 64; + margin_reg_value = 104; + break; + case DP_TRAIN_VOLTAGE_SWING_600: + deemph_reg_value = 64; + margin_reg_value = 154; + break; + default: + return 0; + } + break; + case DP_TRAIN_PRE_EMPHASIS_9_5: + switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { + case DP_TRAIN_VOLTAGE_SWING_400: + deemph_reg_value = 43; + margin_reg_value = 154; + break; + default: + return 0; + } + break; + default: + return 0; + } + + mutex_lock(&dev_priv->dpio_lock); + + /* Clear calc init */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + + /* Program swing deemph */ + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); + val &= ~DPIO_SWING_DEEMPH9P5_MASK; + val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); + } + + /* Program swing margin */ + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); + val &= ~DPIO_SWING_MARGIN_MASK; + val |= margin_reg_value << DPIO_SWING_MARGIN_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); + } + + /* Disable unique transition scale */ + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); + val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); + } + + if (((train_set & DP_TRAIN_PRE_EMPHASIS_MASK) + == DP_TRAIN_PRE_EMPHASIS_0) && + ((train_set & DP_TRAIN_VOLTAGE_SWING_MASK) + == DP_TRAIN_VOLTAGE_SWING_1200)) { + + /* + * The document said it needs to set bit 27 for ch0 and bit 26 + * for ch1. Might be a typo in the doc. + * For now, for this unique transition scale selection, set bit + * 27 for ch0 and ch1. + */ + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); + val |= DPIO_TX_UNIQ_TRANS_SCALE_EN; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); + } + + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); + val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); + val |= (0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT); + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); + } + } + + /* Start swing calculation */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + + /* LRC Bypass */ + val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30); + val |= DPIO_LRC_BYPASS; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, val); + + mutex_unlock(&dev_priv->dpio_lock); + + return 0; +} + static void intel_get_adjust_train(struct intel_dp *intel_dp, const uint8_t link_status[DP_LINK_STATUS_SIZE]) @@ -2346,6 +2750,9 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP) } else if (IS_HASWELL(dev)) { signal_levels = intel_hsw_signal_levels(train_set); mask = DDI_BUF_EMP_MASK; + } else if (IS_CHERRYVIEW(dev)) { + signal_levels = intel_chv_signal_levels(intel_dp); + mask = 0; } else if (IS_VALLEYVIEW(dev)) { signal_levels = intel_vlv_signal_levels(intel_dp); mask = 0; @@ -2456,8 +2863,8 @@ intel_dp_set_link_train(struct intel_dp *intel_dp, len = intel_dp->lane_count + 1; } - ret = intel_dp_aux_native_write(intel_dp, DP_TRAINING_PATTERN_SET, - buf, len); + ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET, + buf, len); return ret == len; } @@ -2486,9 +2893,8 @@ intel_dp_update_link_train(struct intel_dp *intel_dp, uint32_t *DP, I915_WRITE(intel_dp->output_reg, *DP); POSTING_READ(intel_dp->output_reg); - ret = intel_dp_aux_native_write(intel_dp, DP_TRAINING_LANE0_SET, - intel_dp->train_set, - intel_dp->lane_count); + ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET, + intel_dp->train_set, intel_dp->lane_count); return ret == intel_dp->lane_count; } @@ -2544,11 +2950,11 @@ intel_dp_start_link_train(struct intel_dp *intel_dp) link_config[1] = intel_dp->lane_count; if (drm_dp_enhanced_frame_cap(intel_dp->dpcd)) link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN; - intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET, link_config, 2); + drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2); link_config[0] = 0; link_config[1] = DP_SET_ANSI_8B10B; - intel_dp_aux_native_write(intel_dp, DP_DOWNSPREAD_CTRL, link_config, 2); + drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2); DP |= DP_PORT_EN; @@ -2621,10 +3027,15 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp) bool channel_eq = false; int tries, cr_tries; uint32_t DP = intel_dp->DP; + uint32_t training_pattern = DP_TRAINING_PATTERN_2; + + /* Training Pattern 3 for HBR2 ot 1.2 devices that support it*/ + if (intel_dp->link_bw == DP_LINK_BW_5_4 || intel_dp->use_tps3) + training_pattern = DP_TRAINING_PATTERN_3; /* channel equalization */ if (!intel_dp_set_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_2 | + training_pattern | DP_LINK_SCRAMBLING_DISABLE)) { DRM_ERROR("failed to start channel equalization\n"); return; @@ -2651,7 +3062,7 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp) if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { intel_dp_start_link_train(intel_dp); intel_dp_set_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_2 | + training_pattern | DP_LINK_SCRAMBLING_DISABLE); cr_tries++; continue; @@ -2667,7 +3078,7 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp) intel_dp_link_down(intel_dp); intel_dp_start_link_train(intel_dp); intel_dp_set_link_train(intel_dp, &DP, - DP_TRAINING_PATTERN_2 | + training_pattern | DP_LINK_SCRAMBLING_DISABLE); tries = 0; cr_tries++; @@ -2708,22 +3119,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) to_intel_crtc(intel_dig_port->base.base.crtc); uint32_t DP = intel_dp->DP; - /* - * DDI code has a strict mode set sequence and we should try to respect - * it, otherwise we might hang the machine in many different ways. So we - * really should be disabling the port only on a complete crtc_disable - * sequence. This function is just called under two conditions on DDI - * code: - * - Link train failed while doing crtc_enable, and on this case we - * really should respect the mode set sequence and wait for a - * crtc_disable. - * - Someone turned the monitor off and intel_dp_check_link_status - * called us. We don't need to disable the whole port on this case, so - * when someone turns the monitor on again, - * intel_ddi_prepare_link_retrain will take care of redoing the link - * train. - */ - if (HAS_DDI(dev)) + if (WARN_ON(HAS_DDI(dev))) return; if (WARN_ON((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)) @@ -2740,9 +3136,6 @@ intel_dp_link_down(struct intel_dp *intel_dp) } POSTING_READ(intel_dp->output_reg); - /* We don't really know why we're doing this */ - intel_wait_for_vblank(dev, intel_crtc->pipe); - if (HAS_PCH_IBX(dev) && I915_READ(intel_dp->output_reg) & DP_PIPEB_SELECT) { struct drm_crtc *crtc = intel_dig_port->base.base.crtc; @@ -2786,8 +3179,8 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) char dpcd_hex_dump[sizeof(intel_dp->dpcd) * 3]; - if (intel_dp_aux_native_read_retry(intel_dp, 0x000, intel_dp->dpcd, - sizeof(intel_dp->dpcd)) == 0) + if (intel_dp_dpcd_read_wake(&intel_dp->aux, 0x000, intel_dp->dpcd, + sizeof(intel_dp->dpcd)) < 0) return false; /* aux transfer failed */ hex_dump_to_buffer(intel_dp->dpcd, sizeof(intel_dp->dpcd), @@ -2800,15 +3193,23 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) /* Check if the panel supports PSR */ memset(intel_dp->psr_dpcd, 0, sizeof(intel_dp->psr_dpcd)); if (is_edp(intel_dp)) { - intel_dp_aux_native_read_retry(intel_dp, DP_PSR_SUPPORT, - intel_dp->psr_dpcd, - sizeof(intel_dp->psr_dpcd)); + intel_dp_dpcd_read_wake(&intel_dp->aux, DP_PSR_SUPPORT, + intel_dp->psr_dpcd, + sizeof(intel_dp->psr_dpcd)); if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) { dev_priv->psr.sink_support = true; DRM_DEBUG_KMS("Detected EDP PSR Panel.\n"); } } + /* Training Pattern 3 support */ + if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 && + intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED) { + intel_dp->use_tps3 = true; + DRM_DEBUG_KMS("Displayport TPS3 supported"); + } else + intel_dp->use_tps3 = false; + if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT)) return true; /* native DP sink */ @@ -2816,9 +3217,9 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) if (intel_dp->dpcd[DP_DPCD_REV] == 0x10) return true; /* no per-port downstream info */ - if (intel_dp_aux_native_read_retry(intel_dp, DP_DOWNSTREAM_PORT_0, - intel_dp->downstream_ports, - DP_MAX_DOWNSTREAM_PORTS) == 0) + if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_DOWNSTREAM_PORT_0, + intel_dp->downstream_ports, + DP_MAX_DOWNSTREAM_PORTS) < 0) return false; /* downstream port status fetch failed */ return true; @@ -2832,38 +3233,61 @@ intel_dp_probe_oui(struct intel_dp *intel_dp) if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT)) return; - ironlake_edp_panel_vdd_on(intel_dp); + intel_edp_panel_vdd_on(intel_dp); - if (intel_dp_aux_native_read_retry(intel_dp, DP_SINK_OUI, buf, 3)) + if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_SINK_OUI, buf, 3) == 3) DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n", buf[0], buf[1], buf[2]); - if (intel_dp_aux_native_read_retry(intel_dp, DP_BRANCH_OUI, buf, 3)) + if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_BRANCH_OUI, buf, 3) == 3) DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n", buf[0], buf[1], buf[2]); - ironlake_edp_panel_vdd_off(intel_dp, false); + edp_panel_vdd_off(intel_dp, false); } -static bool -intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector) +int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) { - int ret; + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct intel_crtc *intel_crtc = + to_intel_crtc(intel_dig_port->base.base.crtc); + u8 buf[1]; - ret = intel_dp_aux_native_read_retry(intel_dp, - DP_DEVICE_SERVICE_IRQ_VECTOR, - sink_irq_vector, 1); - if (!ret) - return false; + if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, buf) < 0) + return -EAGAIN; - return true; + if (!(buf[0] & DP_TEST_CRC_SUPPORTED)) + return -ENOTTY; + + if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, + DP_TEST_SINK_START) < 0) + return -EAGAIN; + + /* Wait 2 vblanks to be sure we will have the correct CRC value */ + intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev, intel_crtc->pipe); + + if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0) + return -EAGAIN; + + drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, 0); + return 0; +} + +static bool +intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector) +{ + return intel_dp_dpcd_read_wake(&intel_dp->aux, + DP_DEVICE_SERVICE_IRQ_VECTOR, + sink_irq_vector, 1) == 1; } static void intel_dp_handle_test_request(struct intel_dp *intel_dp) { /* NAK by default */ - intel_dp_aux_native_write_1(intel_dp, DP_TEST_RESPONSE, DP_TEST_NAK); + drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_RESPONSE, DP_TEST_NAK); } /* @@ -2882,6 +3306,7 @@ intel_dp_check_link_status(struct intel_dp *intel_dp) u8 sink_irq_vector; u8 link_status[DP_LINK_STATUS_SIZE]; + /* FIXME: This access isn't protected by any locks. */ if (!intel_encoder->connectors_active) return; @@ -2902,9 +3327,9 @@ intel_dp_check_link_status(struct intel_dp *intel_dp) if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && intel_dp_get_sink_irq(intel_dp, &sink_irq_vector)) { /* Clear interrupt source */ - intel_dp_aux_native_write_1(intel_dp, - DP_DEVICE_SERVICE_IRQ_VECTOR, - sink_irq_vector); + drm_dp_dpcd_writeb(&intel_dp->aux, + DP_DEVICE_SERVICE_IRQ_VECTOR, + sink_irq_vector); if (sink_irq_vector & DP_AUTOMATED_TEST_REQUEST) intel_dp_handle_test_request(intel_dp); @@ -2914,7 +3339,7 @@ intel_dp_check_link_status(struct intel_dp *intel_dp) if (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) { DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n", - drm_get_encoder_name(&intel_encoder->base)); + intel_encoder->base.name); intel_dp_start_link_train(intel_dp); intel_dp_complete_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); @@ -2939,15 +3364,17 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp) if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && intel_dp->downstream_ports[0] & DP_DS_PORT_HPD) { uint8_t reg; - if (!intel_dp_aux_native_read_retry(intel_dp, DP_SINK_COUNT, - ®, 1)) + + if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_SINK_COUNT, + ®, 1) < 0) return connector_status_unknown; + return DP_GET_SINK_COUNT(reg) ? connector_status_connected : connector_status_disconnected; } /* If no HPD, poke DDC gently */ - if (drm_probe_ddc(&intel_dp->adapter)) + if (drm_probe_ddc(&intel_dp->aux.ddc)) return connector_status_connected; /* Well we tried, say unknown for unreliable port types */ @@ -3089,12 +3516,16 @@ intel_dp_detect(struct drm_connector *connector, bool force) struct drm_device *dev = connector->dev; struct drm_i915_private *dev_priv = dev->dev_private; enum drm_connector_status status; + enum intel_display_power_domain power_domain; struct edid *edid = NULL; intel_runtime_pm_get(dev_priv); + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); intel_dp->has_audio = false; @@ -3111,7 +3542,7 @@ intel_dp_detect(struct drm_connector *connector, bool force) if (intel_dp->force_audio != HDMI_AUDIO_AUTO) { intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON); } else { - edid = intel_dp_get_edid(connector, &intel_dp->adapter); + edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc); if (edid) { intel_dp->has_audio = drm_detect_monitor_audio(edid); kfree(edid); @@ -3123,21 +3554,32 @@ intel_dp_detect(struct drm_connector *connector, bool force) status = connector_status_connected; out: + intel_display_power_put(dev_priv, power_domain); + intel_runtime_pm_put(dev_priv); + return status; } static int intel_dp_get_modes(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum intel_display_power_domain power_domain; int ret; /* We should parse the EDID data and find out if it has an audio sink */ - ret = intel_dp_get_edid_modes(connector, &intel_dp->adapter); + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + + ret = intel_dp_get_edid_modes(connector, &intel_dp->aux.ddc); + intel_display_power_put(dev_priv, power_domain); if (ret) return ret; @@ -3158,15 +3600,25 @@ static bool intel_dp_detect_audio(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *intel_encoder = &intel_dig_port->base; + struct drm_device *dev = connector->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + enum intel_display_power_domain power_domain; struct edid *edid; bool has_audio = false; - edid = intel_dp_get_edid(connector, &intel_dp->adapter); + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + + edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc); if (edid) { has_audio = drm_detect_monitor_audio(edid); kfree(edid); } + intel_display_power_put(dev_priv, power_domain); + return has_audio; } @@ -3281,13 +3733,17 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder) struct intel_dp *intel_dp = &intel_dig_port->dp; struct drm_device *dev = intel_dp_to_dev(intel_dp); - i2c_del_adapter(&intel_dp->adapter); + drm_dp_aux_unregister(&intel_dp->aux); drm_encoder_cleanup(encoder); if (is_edp(intel_dp)) { cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - mutex_lock(&dev->mode_config.mutex); - ironlake_panel_vdd_off_sync(intel_dp); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + edp_panel_vdd_off_sync(intel_dp); + drm_modeset_unlock(&dev->mode_config.connection_mutex); + if (intel_dp->edp_notifier.notifier_call) { + unregister_reboot_notifier(&intel_dp->edp_notifier); + intel_dp->edp_notifier.notifier_call = NULL; + } } kfree(intel_dig_port); } @@ -3385,6 +3841,13 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect } } +static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp) +{ + intel_dp->last_power_cycle = jiffies; + intel_dp->last_power_on = jiffies; + intel_dp->last_backlight_off = jiffies; +} + static void intel_dp_init_panel_power_sequencer(struct drm_device *dev, struct intel_dp *intel_dp, @@ -3507,10 +3970,17 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe); } - /* And finally store the new values in the power sequencer. */ + /* + * And finally store the new values in the power sequencer. The + * backlight delays are set to 1 because we do manual waits on them. For + * T8, even BSpec recommends doing it. For T9, if we don't do this, + * we'll end up waiting for the backlight off delay twice: once when we + * do the manual sleep, and once when we disable the panel and wait for + * the PP_STATUS bit to become zero. + */ pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) | - (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT); - pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) | + (1 << PANEL_LIGHT_ON_DELAY_SHIFT); + pp_off = (1 << PANEL_LIGHT_OFF_DELAY_SHIFT) | (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT); /* Compute the divisor for the pp clock, simply match the Bspec * formula. */ @@ -3544,28 +4014,162 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, I915_READ(pp_div_reg)); } +void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_encoder *encoder; + struct intel_dp *intel_dp = NULL; + struct intel_crtc_config *config = NULL; + struct intel_crtc *intel_crtc = NULL; + struct intel_connector *intel_connector = dev_priv->drrs.connector; + u32 reg, val; + enum edp_drrs_refresh_rate_type index = DRRS_HIGH_RR; + + if (refresh_rate <= 0) { + DRM_DEBUG_KMS("Refresh rate should be positive non-zero.\n"); + return; + } + + if (intel_connector == NULL) { + DRM_DEBUG_KMS("DRRS supported for eDP only.\n"); + return; + } + + if (INTEL_INFO(dev)->gen < 8 && intel_edp_is_psr_enabled(dev)) { + DRM_DEBUG_KMS("DRRS is disabled as PSR is enabled\n"); + return; + } + + encoder = intel_attached_encoder(&intel_connector->base); + intel_dp = enc_to_intel_dp(&encoder->base); + intel_crtc = encoder->new_crtc; + + if (!intel_crtc) { + DRM_DEBUG_KMS("DRRS: intel_crtc not initialized\n"); + return; + } + + config = &intel_crtc->config; + + if (intel_dp->drrs_state.type < SEAMLESS_DRRS_SUPPORT) { + DRM_DEBUG_KMS("Only Seamless DRRS supported.\n"); + return; + } + + if (intel_connector->panel.downclock_mode->vrefresh == refresh_rate) + index = DRRS_LOW_RR; + + if (index == intel_dp->drrs_state.refresh_rate_type) { + DRM_DEBUG_KMS( + "DRRS requested for previously set RR...ignoring\n"); + return; + } + + if (!intel_crtc->active) { + DRM_DEBUG_KMS("eDP encoder disabled. CRTC not Active\n"); + return; + } + + if (INTEL_INFO(dev)->gen > 6 && INTEL_INFO(dev)->gen < 8) { + reg = PIPECONF(intel_crtc->config.cpu_transcoder); + val = I915_READ(reg); + if (index > DRRS_HIGH_RR) { + val |= PIPECONF_EDP_RR_MODE_SWITCH; + intel_dp_set_m2_n2(intel_crtc, &config->dp_m2_n2); + } else { + val &= ~PIPECONF_EDP_RR_MODE_SWITCH; + } + I915_WRITE(reg, val); + } + + /* + * mutex taken to ensure that there is no race between differnt + * drrs calls trying to update refresh rate. This scenario may occur + * in future when idleness detection based DRRS in kernel and + * possible calls from user space to set differnt RR are made. + */ + + mutex_lock(&intel_dp->drrs_state.mutex); + + intel_dp->drrs_state.refresh_rate_type = index; + + mutex_unlock(&intel_dp->drrs_state.mutex); + + DRM_DEBUG_KMS("eDP Refresh Rate set to : %dHz\n", refresh_rate); +} + +static struct drm_display_mode * +intel_dp_drrs_init(struct intel_digital_port *intel_dig_port, + struct intel_connector *intel_connector, + struct drm_display_mode *fixed_mode) +{ + struct drm_connector *connector = &intel_connector->base; + struct intel_dp *intel_dp = &intel_dig_port->dp; + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_display_mode *downclock_mode = NULL; + + if (INTEL_INFO(dev)->gen <= 6) { + DRM_DEBUG_KMS("DRRS supported for Gen7 and above\n"); + return NULL; + } + + if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) { + DRM_INFO("VBT doesn't support DRRS\n"); + return NULL; + } + + downclock_mode = intel_find_panel_downclock + (dev, fixed_mode, connector); + + if (!downclock_mode) { + DRM_INFO("DRRS not supported\n"); + return NULL; + } + + dev_priv->drrs.connector = intel_connector; + + mutex_init(&intel_dp->drrs_state.mutex); + + intel_dp->drrs_state.type = dev_priv->vbt.drrs_type; + + intel_dp->drrs_state.refresh_rate_type = DRRS_HIGH_RR; + DRM_INFO("seamless DRRS supported for eDP panel.\n"); + return downclock_mode; +} + static bool intel_edp_init_connector(struct intel_dp *intel_dp, - struct intel_connector *intel_connector) + struct intel_connector *intel_connector, + struct edp_power_seq *power_seq) { struct drm_connector *connector = &intel_connector->base; struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; + struct intel_encoder *intel_encoder = &intel_dig_port->base; + struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_display_mode *fixed_mode = NULL; - struct edp_power_seq power_seq = { 0 }; + struct drm_display_mode *downclock_mode = NULL; bool has_dpcd; struct drm_display_mode *scan; struct edid *edid; + intel_dp->drrs_state.type = DRRS_NOT_SUPPORTED; + if (!is_edp(intel_dp)) return true; - intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + /* The VDD bit needs a power domain reference, so if the bit is already + * enabled when we boot, grab this reference. */ + if (edp_have_panel_vdd(intel_dp)) { + enum intel_display_power_domain power_domain; + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + } /* Cache DPCD and EDID for edp. */ - ironlake_edp_panel_vdd_on(intel_dp); + intel_edp_panel_vdd_on(intel_dp); has_dpcd = intel_dp_get_dpcd(intel_dp); - ironlake_edp_panel_vdd_off(intel_dp, false); + edp_panel_vdd_off(intel_dp, false); if (has_dpcd) { if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) @@ -3579,10 +4183,10 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, } /* We now know it's not a ghost, init power sequence regs. */ - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, - &power_seq); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, power_seq); - edid = drm_get_edid(connector, &intel_dp->adapter); + mutex_lock(&dev->mode_config.mutex); + edid = drm_get_edid(connector, &intel_dp->aux.ddc); if (edid) { if (drm_add_edid_modes(connector, edid)) { drm_mode_connector_update_edid_property(connector, @@ -3601,6 +4205,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, list_for_each_entry(scan, &connector->probed_modes, head) { if ((scan->type & DRM_MODE_TYPE_PREFERRED)) { fixed_mode = drm_mode_duplicate(dev, scan); + downclock_mode = intel_dp_drrs_init( + intel_dig_port, + intel_connector, fixed_mode); break; } } @@ -3612,8 +4219,14 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, if (fixed_mode) fixed_mode->type |= DRM_MODE_TYPE_PREFERRED; } + mutex_unlock(&dev->mode_config.mutex); + + if (IS_VALLEYVIEW(dev)) { + intel_dp->edp_notifier.notifier_call = edp_notify_handler; + register_reboot_notifier(&intel_dp->edp_notifier); + } - intel_panel_init(&intel_connector->panel, fixed_mode); + intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); intel_panel_setup_backlight(connector); return true; @@ -3629,8 +4242,20 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum port port = intel_dig_port->port; - const char *name = NULL; - int type, error; + struct edp_power_seq power_seq = { 0 }; + int type; + + /* intel_dp vfuncs */ + if (IS_VALLEYVIEW(dev)) + intel_dp->get_aux_clock_divider = vlv_get_aux_clock_divider; + else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider; + else if (HAS_PCH_SPLIT(dev)) + intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider; + else + intel_dp->get_aux_clock_divider = i9xx_get_aux_clock_divider; + + intel_dp->get_aux_send_ctl = i9xx_get_aux_send_ctl; /* Preserve the current hw state. */ intel_dp->DP = I915_READ(intel_dp->output_reg); @@ -3660,7 +4285,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, connector->doublescan_allowed = 0; INIT_DELAYED_WORK(&intel_dp->panel_vdd_work, - ironlake_panel_vdd_work); + edp_panel_vdd_work); intel_connector_attach_encoder(intel_connector, intel_encoder); drm_sysfs_connector_add(connector); @@ -3669,62 +4294,42 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; else intel_connector->get_hw_state = intel_connector_get_hw_state; + intel_connector->unregister = intel_dp_connector_unregister; - intel_dp->aux_ch_ctl_reg = intel_dp->output_reg + 0x10; - if (HAS_DDI(dev)) { - switch (intel_dig_port->port) { - case PORT_A: - intel_dp->aux_ch_ctl_reg = DPA_AUX_CH_CTL; - break; - case PORT_B: - intel_dp->aux_ch_ctl_reg = PCH_DPB_AUX_CH_CTL; - break; - case PORT_C: - intel_dp->aux_ch_ctl_reg = PCH_DPC_AUX_CH_CTL; - break; - case PORT_D: - intel_dp->aux_ch_ctl_reg = PCH_DPD_AUX_CH_CTL; - break; - default: - BUG(); - } - } - - /* Set up the DDC bus. */ + /* Set up the hotplug pin. */ switch (port) { case PORT_A: intel_encoder->hpd_pin = HPD_PORT_A; - name = "DPDDC-A"; break; case PORT_B: intel_encoder->hpd_pin = HPD_PORT_B; - name = "DPDDC-B"; break; case PORT_C: intel_encoder->hpd_pin = HPD_PORT_C; - name = "DPDDC-C"; break; case PORT_D: intel_encoder->hpd_pin = HPD_PORT_D; - name = "DPDDC-D"; break; default: BUG(); } - error = intel_dp_i2c_init(intel_dp, intel_connector, name); - WARN(error, "intel_dp_i2c_init failed with error %d for port %c\n", - error, port_name(port)); + if (is_edp(intel_dp)) { + intel_dp_init_panel_power_timestamps(intel_dp); + intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); + } + + intel_dp_aux_init(intel_dp, intel_connector); intel_dp->psr_setup_done = false; - if (!intel_edp_init_connector(intel_dp, intel_connector)) { - i2c_del_adapter(&intel_dp->adapter); + if (!intel_edp_init_connector(intel_dp, intel_connector, &power_seq)) { + drm_dp_aux_unregister(&intel_dp->aux); if (is_edp(intel_dp)) { cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - mutex_lock(&dev->mode_config.mutex); - ironlake_panel_vdd_off_sync(intel_dp); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); + edp_panel_vdd_off_sync(intel_dp); + drm_modeset_unlock(&dev->mode_config.connection_mutex); } drm_sysfs_connector_remove(connector); drm_connector_cleanup(connector); @@ -3770,26 +4375,37 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port) DRM_MODE_ENCODER_TMDS); intel_encoder->compute_config = intel_dp_compute_config; - intel_encoder->mode_set = intel_dp_mode_set; intel_encoder->disable = intel_disable_dp; - intel_encoder->post_disable = intel_post_disable_dp; intel_encoder->get_hw_state = intel_dp_get_hw_state; intel_encoder->get_config = intel_dp_get_config; - if (IS_VALLEYVIEW(dev)) { + if (IS_CHERRYVIEW(dev)) { + intel_encoder->pre_enable = chv_pre_enable_dp; + intel_encoder->enable = vlv_enable_dp; + intel_encoder->post_disable = chv_post_disable_dp; + } else if (IS_VALLEYVIEW(dev)) { intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable; intel_encoder->pre_enable = vlv_pre_enable_dp; intel_encoder->enable = vlv_enable_dp; + intel_encoder->post_disable = vlv_post_disable_dp; } else { intel_encoder->pre_enable = g4x_pre_enable_dp; intel_encoder->enable = g4x_enable_dp; + intel_encoder->post_disable = g4x_post_disable_dp; } intel_dig_port->port = port; intel_dig_port->dp.output_reg = output_reg; intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; - intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); - intel_encoder->cloneable = false; + if (IS_CHERRYVIEW(dev)) { + if (port == PORT_D) + intel_encoder->crtc_mask = 1 << 2; + else + intel_encoder->crtc_mask = (1 << 0) | (1 << 1); + } else { + intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); + } + intel_encoder->cloneable = 0; intel_encoder->hot_plug = intel_dp_hot_plug; if (!intel_dp_init_connector(intel_dig_port, intel_connector)) { diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index fbfaaba5cc3..f67340ed2c1 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -78,6 +78,12 @@ #define MAX_OUTPUTS 6 /* maximum connectors per crtcs in the mode set */ +/* Maximum cursor sizes */ +#define GEN2_CURSOR_WIDTH 64 +#define GEN2_CURSOR_HEIGHT 64 +#define MAX_CURSOR_WIDTH 256 +#define MAX_CURSOR_HEIGHT 256 + #define INTEL_I2C_BUS_DVO 1 #define INTEL_I2C_BUS_SDVO 2 @@ -100,8 +106,8 @@ #define INTEL_DVO_CHIP_TMDS 2 #define INTEL_DVO_CHIP_TVOUT 4 -#define INTEL_DSI_COMMAND_MODE 0 -#define INTEL_DSI_VIDEO_MODE 1 +#define INTEL_DSI_VIDEO_MODE 0 +#define INTEL_DSI_COMMAND_MODE 1 struct intel_framebuffer { struct drm_framebuffer base; @@ -110,9 +116,10 @@ struct intel_framebuffer { struct intel_fbdev { struct drm_fb_helper helper; - struct intel_framebuffer ifb; + struct intel_framebuffer *fb; struct list_head fbdev_list; struct drm_display_mode *our_mode; + int preferred_bpp; }; struct intel_encoder { @@ -124,11 +131,7 @@ struct intel_encoder { struct intel_crtc *new_crtc; int type; - /* - * Intel hw has only one MUX where encoders could be clone, hence a - * simple flag is enough to compute the possible_clones mask. - */ - bool cloneable; + unsigned int cloneable; bool connectors_active; void (*hot_plug)(struct intel_encoder *); bool (*compute_config)(struct intel_encoder *, @@ -187,6 +190,14 @@ struct intel_connector { * and active (i.e. dpms ON state). */ bool (*get_hw_state)(struct intel_connector *); + /* + * Removes all interfaces through which the connector is accessible + * - like sysfs, debugfs entries -, so that no new operations can be + * started on the connector. Also makes sure all currently pending + * operations finish before returing. + */ + void (*unregister)(struct intel_connector *); + /* Panel info for eDP and LVDS */ struct intel_panel panel; @@ -210,6 +221,12 @@ typedef struct dpll { int p; } intel_clock_t; +struct intel_plane_config { + bool tiled; + int size; + u32 base; +}; + struct intel_crtc_config { /** * quirks - bitfield with hw state readout quirks @@ -219,7 +236,8 @@ struct intel_crtc_config { * tracked with quirk flags so that fastboot and state checker can act * accordingly. */ -#define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */ +#define PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS (1<<0) /* unreliable sync mode.flags */ +#define PIPE_CONFIG_QUIRK_INHERITED_MODE (1<<1) /* mode inherited from firmware */ unsigned long quirks; /* User requested mode, only valid as a starting point to @@ -255,6 +273,13 @@ struct intel_crtc_config { * accordingly. */ bool has_dp_encoder; + /* Whether we should send NULL infoframes. Required for audio. */ + bool has_hdmi_sink; + + /* Audio enabled on this pipe. Only valid if either has_hdmi_sink or + * has_dp_encoder is set. */ + bool has_audio; + /* * Enable dithering, used when the selected pipe bpp doesn't match the * plane bpp. @@ -288,6 +313,9 @@ struct intel_crtc_config { int pipe_bpp; struct intel_link_m_n dp_m_n; + /* m2_n2 for eDP downclock */ + struct intel_link_m_n dp_m2_n2; + /* * Frequence the dpll for the port should run at. Differs from the * adjusted dotclock e.g. for DP or 12bpc hdmi mode. This is also @@ -325,6 +353,9 @@ struct intel_pipe_wm { struct intel_wm_level wm[5]; uint32_t linetime; bool fbc_wm_enabled; + bool pipe_enabled; + bool sprites_enabled; + bool sprites_scaled; }; struct intel_crtc { @@ -339,7 +370,6 @@ struct intel_crtc { */ bool active; unsigned long enabled_power_domains; - bool eld_vld; bool primary_enabled; /* is the primary plane (partially) visible? */ bool lowfreq_avail; struct intel_overlay *overlay; @@ -356,9 +386,13 @@ struct intel_crtc { uint32_t cursor_addr; int16_t cursor_x, cursor_y; int16_t cursor_width, cursor_height; - bool cursor_visible; + uint32_t cursor_cntl; + uint32_t cursor_base; + struct intel_plane_config plane_config; struct intel_crtc_config config; + struct intel_crtc_config *new_config; + bool new_enabled; uint32_t ddi_pll_sel; @@ -374,6 +408,10 @@ struct intel_crtc { /* watermarks currently being used */ struct intel_pipe_wm active; } wm; + + wait_queue_head_t vbl_wait; + + int scanline_offset; }; struct intel_plane_wm_parameters { @@ -457,11 +495,23 @@ struct intel_hdmi { enum hdmi_infoframe_type type, const void *frame, ssize_t len); void (*set_infoframes)(struct drm_encoder *encoder, + bool enable, struct drm_display_mode *adjusted_mode); }; #define DP_MAX_DOWNSTREAM_PORTS 0x10 +/** + * HIGH_RR is the highest eDP panel refresh rate read from EDID + * LOW_RR is the lowest eDP panel refresh rate found from EDID + * parsing for same resolution. + */ +enum edp_drrs_refresh_rate_type { + DRRS_HIGH_RR, + DRRS_LOW_RR, + DRRS_MAX_RR, /* RR count */ +}; + struct intel_dp { uint32_t output_reg; uint32_t aux_ch_ctl_reg; @@ -475,8 +525,7 @@ struct intel_dp { uint8_t dpcd[DP_RECEIVER_CAP_SIZE]; uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; - struct i2c_adapter adapter; - struct i2c_algo_dp_aux_data algo; + struct drm_dp_aux aux; uint8_t train_set[4]; int panel_power_up_delay; int panel_power_down_delay; @@ -485,8 +534,30 @@ struct intel_dp { int backlight_off_delay; struct delayed_work panel_vdd_work; bool want_panel_vdd; + unsigned long last_power_cycle; + unsigned long last_power_on; + unsigned long last_backlight_off; bool psr_setup_done; + struct notifier_block edp_notifier; + + bool use_tps3; struct intel_connector *attached_connector; + + uint32_t (*get_aux_clock_divider)(struct intel_dp *dp, int index); + /* + * This function returns the value we have to program the AUX_CTL + * register with to kick off an AUX transaction. + */ + uint32_t (*get_aux_send_ctl)(struct intel_dp *dp, + bool has_aux_irq, + int send_bytes, + uint32_t aux_clock_divider); + struct { + enum drrs_support_type type; + enum edp_drrs_refresh_rate_type refresh_rate_type; + struct mutex mutex; + } drrs_state; + }; struct intel_digital_port { @@ -502,6 +573,7 @@ vlv_dport_to_channel(struct intel_digital_port *dport) { switch (dport->port) { case PORT_B: + case PORT_D: return DPIO_CH0; case PORT_C: return DPIO_CH1; @@ -510,6 +582,20 @@ vlv_dport_to_channel(struct intel_digital_port *dport) } } +static inline int +vlv_pipe_to_channel(enum pipe pipe) +{ + switch (pipe) { + case PIPE_A: + case PIPE_C: + return DPIO_CH0; + case PIPE_B: + return DPIO_CH1; + default: + BUG(); + } +} + static inline struct drm_crtc * intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) { @@ -534,12 +620,15 @@ struct intel_unpin_work { #define INTEL_FLIP_INACTIVE 0 #define INTEL_FLIP_PENDING 1 #define INTEL_FLIP_COMPLETE 2 + u32 flip_count; + u32 gtt_offset; bool enable_stall_check; }; struct intel_set_config { struct drm_encoder **save_connector_encoders; struct drm_crtc **save_encoder_crtcs; + bool *save_crtc_enabled; bool fb_changed; bool mode_changed; @@ -591,8 +680,12 @@ void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); -void hsw_pc8_disable_interrupts(struct drm_device *dev); -void hsw_pc8_restore_interrupts(struct drm_device *dev); +void bdw_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); +void bdw_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); +void intel_runtime_pm_disable_interrupts(struct drm_device *dev); +void intel_runtime_pm_restore_interrupts(struct drm_device *dev); +int intel_get_crtc_scanline(struct intel_crtc *crtc); +void i9xx_check_fifo_underruns(struct drm_device *dev); /* intel_crt.c */ @@ -628,9 +721,10 @@ void intel_ddi_get_config(struct intel_encoder *encoder, const char *intel_output_name(int output); bool intel_has_pending_fb_unpin(struct drm_device *dev); int intel_pch_rawclk(struct drm_device *dev); +int valleyview_cur_cdclk(struct drm_i915_private *dev_priv); void intel_mark_busy(struct drm_device *dev); void intel_mark_fb_busy(struct drm_i915_gem_object *obj, - struct intel_ring_buffer *ring); + struct intel_engine_cs *ring); void intel_mark_idle(struct drm_device *dev); void intel_crtc_restore_mode(struct drm_crtc *crtc); void intel_crtc_update_dpms(struct drm_crtc *crtc); @@ -657,18 +751,19 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, struct intel_digital_port *dport); bool intel_get_load_detect_pipe(struct drm_connector *connector, struct drm_display_mode *mode, - struct intel_load_detect_pipe *old); + struct intel_load_detect_pipe *old, + struct drm_modeset_acquire_ctx *ctx); void intel_release_load_detect_pipe(struct drm_connector *connector, - struct intel_load_detect_pipe *old); + struct intel_load_detect_pipe *old, + struct drm_modeset_acquire_ctx *ctx); int intel_pin_and_fence_fb_obj(struct drm_device *dev, struct drm_i915_gem_object *obj, - struct intel_ring_buffer *pipelined); + struct intel_engine_cs *pipelined); void intel_unpin_fb_obj(struct drm_i915_gem_object *obj); -int intel_framebuffer_init(struct drm_device *dev, - struct intel_framebuffer *ifb, +struct drm_framebuffer * +__intel_framebuffer_create(struct drm_device *dev, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_i915_gem_object *obj); -void intel_framebuffer_fini(struct intel_framebuffer *fb); void intel_prepare_page_flip(struct drm_device *dev, int plane); void intel_finish_page_flip(struct drm_device *dev, int pipe); void intel_finish_page_flip_plane(struct drm_device *dev, int plane); @@ -696,9 +791,8 @@ unsigned long intel_gen4_compute_page_offset(int *x, int *y, unsigned int bpp, unsigned int pitch); void intel_display_handle_reset(struct drm_device *dev); -void hsw_enable_pc8_work(struct work_struct *__work); -void hsw_enable_package_c8(struct drm_i915_private *dev_priv); -void hsw_disable_package_c8(struct drm_i915_private *dev_priv); +void hsw_enable_pc8(struct drm_i915_private *dev_priv); +void hsw_disable_pc8(struct drm_i915_private *dev_priv); void intel_dp_get_m_n(struct intel_crtc *crtc, struct intel_crtc_config *pipe_config); int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n); @@ -708,8 +802,15 @@ ironlake_check_encoder_dotclock(const struct intel_crtc_config *pipe_config, bool intel_crtc_active(struct drm_crtc *crtc); void hsw_enable_ips(struct intel_crtc *crtc); void hsw_disable_ips(struct intel_crtc *crtc); -void intel_display_set_init_power(struct drm_device *dev, bool enable); +void intel_display_set_init_power(struct drm_i915_private *dev, bool enable); +enum intel_display_power_domain +intel_display_port_power_domain(struct intel_encoder *intel_encoder); int valleyview_get_vco(struct drm_i915_private *dev_priv); +void intel_mode_from_pipe_config(struct drm_display_mode *mode, + struct intel_crtc_config *pipe_config); +int intel_format_to_fourcc(int format); +void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc); + /* intel_dp.c */ void intel_dp_init(struct drm_device *dev, int output_reg, enum port port); @@ -721,19 +822,19 @@ void intel_dp_stop_link_train(struct intel_dp *intel_dp); void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode); void intel_dp_encoder_destroy(struct drm_encoder *encoder); void intel_dp_check_link_status(struct intel_dp *intel_dp); +int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc); bool intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_config *pipe_config); bool intel_dp_is_edp(struct drm_device *dev, enum port port); -void ironlake_edp_backlight_on(struct intel_dp *intel_dp); -void ironlake_edp_backlight_off(struct intel_dp *intel_dp); -void ironlake_edp_panel_on(struct intel_dp *intel_dp); -void ironlake_edp_panel_off(struct intel_dp *intel_dp); -void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp); -void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync); +void intel_edp_backlight_on(struct intel_dp *intel_dp); +void intel_edp_backlight_off(struct intel_dp *intel_dp); +void intel_edp_panel_vdd_on(struct intel_dp *intel_dp); +void intel_edp_panel_on(struct intel_dp *intel_dp); +void intel_edp_panel_off(struct intel_dp *intel_dp); void intel_edp_psr_enable(struct intel_dp *intel_dp); void intel_edp_psr_disable(struct intel_dp *intel_dp); void intel_edp_psr_update(struct drm_device *dev); - +void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate); /* intel_dsi.c */ bool intel_dsi_init(struct drm_device *dev); @@ -808,7 +909,8 @@ int intel_overlay_attrs(struct drm_device *dev, void *data, /* intel_panel.c */ int intel_panel_init(struct intel_panel *panel, - struct drm_display_mode *fixed_mode); + struct drm_display_mode *fixed_mode, + struct drm_display_mode *downclock_mode); void intel_panel_fini(struct intel_panel *panel); void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, struct drm_display_mode *adjusted_mode); @@ -834,6 +936,7 @@ extern struct drm_display_mode *intel_find_panel_downclock( /* intel_pm.c */ void intel_init_clock_gating(struct drm_device *dev); void intel_suspend_hw(struct drm_device *dev); +int ilk_wm_max_level(const struct drm_device *dev); void intel_update_watermarks(struct drm_crtc *crtc); void intel_update_sprite_watermarks(struct drm_plane *plane, struct drm_crtc *crtc, @@ -845,20 +948,22 @@ bool intel_fbc_enabled(struct drm_device *dev); void intel_update_fbc(struct drm_device *dev); void intel_gpu_ips_init(struct drm_i915_private *dev_priv); void intel_gpu_ips_teardown(void); -int intel_power_domains_init(struct drm_device *dev); -void intel_power_domains_remove(struct drm_device *dev); -bool intel_display_power_enabled(struct drm_device *dev, +int intel_power_domains_init(struct drm_i915_private *); +void intel_power_domains_remove(struct drm_i915_private *); +bool intel_display_power_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); -bool intel_display_power_enabled_sw(struct drm_device *dev, - enum intel_display_power_domain domain); -void intel_display_power_get(struct drm_device *dev, +bool intel_display_power_enabled_unlocked(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain); +void intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); -void intel_display_power_put(struct drm_device *dev, +void intel_display_power_put(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); -void intel_power_domains_init_hw(struct drm_device *dev); -void intel_set_power_well(struct drm_device *dev, bool enable); +void intel_power_domains_init_hw(struct drm_i915_private *dev_priv); +void intel_init_gt_powersave(struct drm_device *dev); +void intel_cleanup_gt_powersave(struct drm_device *dev); void intel_enable_gt_powersave(struct drm_device *dev); void intel_disable_gt_powersave(struct drm_device *dev); +void intel_reset_gt_powersave(struct drm_device *dev); void ironlake_teardown_rc6(struct drm_device *dev); void gen6_update_ring_freq(struct drm_device *dev); void gen6_rps_idle(struct drm_i915_private *dev_priv); @@ -866,11 +971,13 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv); void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv); void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv); void intel_runtime_pm_get(struct drm_i915_private *dev_priv); +void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv); void intel_runtime_pm_put(struct drm_i915_private *dev_priv); void intel_init_runtime_pm(struct drm_i915_private *dev_priv); void intel_fini_runtime_pm(struct drm_i915_private *dev_priv); void ilk_wm_get_hw_state(struct drm_device *dev); - +void __vlv_set_power_well(struct drm_i915_private *dev_priv, + enum punit_power_well power_well_id, bool enable); /* intel_sdvo.c */ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob); diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c index fabbf0d895c..3fd082933c8 100644 --- a/drivers/gpu/drm/i915/intel_dsi.c +++ b/drivers/gpu/drm/i915/intel_dsi.c @@ -35,6 +35,11 @@ /* the sub-encoders aka panel drivers */ static const struct intel_dsi_device intel_dsi_devices[] = { + { + .panel_id = MIPI_DSI_GENERIC_PANEL_ID, + .name = "vbt-generic-dsi-vid-mode-display", + .dev_ops = &vbt_generic_dsi_display_ops, + }, }; static void band_gap_reset(struct drm_i915_private *dev_priv) @@ -59,12 +64,12 @@ static struct intel_dsi *intel_attached_dsi(struct drm_connector *connector) static inline bool is_vid_mode(struct intel_dsi *intel_dsi) { - return intel_dsi->dev.type == INTEL_DSI_VIDEO_MODE; + return intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE; } static inline bool is_cmd_mode(struct intel_dsi *intel_dsi) { - return intel_dsi->dev.type == INTEL_DSI_COMMAND_MODE; + return intel_dsi->operation_mode == INTEL_DSI_COMMAND_MODE; } static void intel_dsi_hot_plug(struct intel_encoder *encoder) @@ -94,13 +99,6 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, return true; } -static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder) -{ - DRM_DEBUG_KMS("\n"); - - vlv_enable_dsi_pll(encoder); -} - static void intel_dsi_device_ready(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; @@ -110,32 +108,27 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder) DRM_DEBUG_KMS("\n"); + mutex_lock(&dev_priv->dpio_lock); + /* program rcomp for compliance, reduce from 50 ohms to 45 ohms + * needed everytime after power gate */ + vlv_flisdsi_write(dev_priv, 0x04, 0x0004); + mutex_unlock(&dev_priv->dpio_lock); + + /* bandgap reset is needed after everytime we do power gate */ + band_gap_reset(dev_priv); + + I915_WRITE(MIPI_DEVICE_READY(pipe), ULPS_STATE_ENTER); + usleep_range(2500, 3000); + val = I915_READ(MIPI_PORT_CTRL(pipe)); I915_WRITE(MIPI_PORT_CTRL(pipe), val | LP_OUTPUT_HOLD); usleep_range(1000, 1500); - I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY | ULPS_STATE_EXIT); - usleep_range(2000, 2500); - I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY); - usleep_range(2000, 2500); - I915_WRITE(MIPI_DEVICE_READY(pipe), 0x00); - usleep_range(2000, 2500); - I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY); - usleep_range(2000, 2500); -} -static void intel_dsi_pre_enable(struct intel_encoder *encoder) -{ - struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); - - DRM_DEBUG_KMS("\n"); - if (intel_dsi->dev.dev_ops->panel_reset) - intel_dsi->dev.dev_ops->panel_reset(&intel_dsi->dev); - - /* put device in ready state */ - intel_dsi_device_ready(encoder); + I915_WRITE(MIPI_DEVICE_READY(pipe), ULPS_STATE_EXIT); + usleep_range(2500, 3000); - if (intel_dsi->dev.dev_ops->send_otp_cmds) - intel_dsi->dev.dev_ops->send_otp_cmds(&intel_dsi->dev); + I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY); + usleep_range(2500, 3000); } static void intel_dsi_enable(struct intel_encoder *encoder) @@ -153,18 +146,78 @@ static void intel_dsi_enable(struct intel_encoder *encoder) I915_WRITE(MIPI_MAX_RETURN_PKT_SIZE(pipe), 8 * 4); else { msleep(20); /* XXX */ - dpi_send_cmd(intel_dsi, TURN_ON); + dpi_send_cmd(intel_dsi, TURN_ON, DPI_LP_MODE_EN); msleep(100); + if (intel_dsi->dev.dev_ops->enable) + intel_dsi->dev.dev_ops->enable(&intel_dsi->dev); + /* assert ip_tg_enable signal */ temp = I915_READ(MIPI_PORT_CTRL(pipe)) & ~LANE_CONFIGURATION_MASK; temp = temp | intel_dsi->port_bits; I915_WRITE(MIPI_PORT_CTRL(pipe), temp | DPI_ENABLE); POSTING_READ(MIPI_PORT_CTRL(pipe)); } +} + +static void intel_dsi_pre_enable(struct intel_encoder *encoder) +{ + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + enum pipe pipe = intel_crtc->pipe; + u32 tmp; + + DRM_DEBUG_KMS("\n"); - if (intel_dsi->dev.dev_ops->enable) - intel_dsi->dev.dev_ops->enable(&intel_dsi->dev); + /* Disable DPOunit clock gating, can stall pipe + * and we need DPLL REFA always enabled */ + tmp = I915_READ(DPLL(pipe)); + tmp |= DPLL_REFA_CLK_ENABLE_VLV; + I915_WRITE(DPLL(pipe), tmp); + + tmp = I915_READ(DSPCLK_GATE_D); + tmp |= DPOUNIT_CLOCK_GATE_DISABLE; + I915_WRITE(DSPCLK_GATE_D, tmp); + + /* put device in ready state */ + intel_dsi_device_ready(encoder); + + msleep(intel_dsi->panel_on_delay); + + if (intel_dsi->dev.dev_ops->panel_reset) + intel_dsi->dev.dev_ops->panel_reset(&intel_dsi->dev); + + if (intel_dsi->dev.dev_ops->send_otp_cmds) + intel_dsi->dev.dev_ops->send_otp_cmds(&intel_dsi->dev); + + /* Enable port in pre-enable phase itself because as per hw team + * recommendation, port should be enabled befor plane & pipe */ + intel_dsi_enable(encoder); +} + +static void intel_dsi_enable_nop(struct intel_encoder *encoder) +{ + DRM_DEBUG_KMS("\n"); + + /* for DSI port enable has to be done before pipe + * and plane enable, so port enable is done in + * pre_enable phase itself unlike other encoders + */ +} + +static void intel_dsi_pre_disable(struct intel_encoder *encoder) +{ + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + + DRM_DEBUG_KMS("\n"); + + if (is_vid_mode(intel_dsi)) { + /* Send Shutdown command to the panel in LP mode */ + dpi_send_cmd(intel_dsi, SHUTDOWN, DPI_LP_MODE_EN); + msleep(10); + } } static void intel_dsi_disable(struct intel_encoder *encoder) @@ -179,9 +232,6 @@ static void intel_dsi_disable(struct intel_encoder *encoder) DRM_DEBUG_KMS("\n"); if (is_vid_mode(intel_dsi)) { - dpi_send_cmd(intel_dsi, SHUTDOWN); - msleep(10); - /* de-assert ip_tg_enable signal */ temp = I915_READ(MIPI_PORT_CTRL(pipe)); I915_WRITE(MIPI_PORT_CTRL(pipe), temp & ~DPI_ENABLE); @@ -190,6 +240,23 @@ static void intel_dsi_disable(struct intel_encoder *encoder) msleep(2); } + /* Panel commands can be sent when clock is in LP11 */ + I915_WRITE(MIPI_DEVICE_READY(pipe), 0x0); + + temp = I915_READ(MIPI_CTRL(pipe)); + temp &= ~ESCAPE_CLOCK_DIVIDER_MASK; + I915_WRITE(MIPI_CTRL(pipe), temp | + intel_dsi->escape_clk_div << + ESCAPE_CLOCK_DIVIDER_SHIFT); + + I915_WRITE(MIPI_EOT_DISABLE(pipe), CLOCKSTOP); + + temp = I915_READ(MIPI_DSI_FUNC_PRG(pipe)); + temp &= ~VID_MODE_FORMAT_MASK; + I915_WRITE(MIPI_DSI_FUNC_PRG(pipe), temp); + + I915_WRITE(MIPI_DEVICE_READY(pipe), 0x1); + /* if disable packets are sent before sending shutdown packet then in * some next enable sequence send turn on packet error is observed */ if (intel_dsi->dev.dev_ops->disable) @@ -205,49 +272,66 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) DRM_DEBUG_KMS("\n"); - I915_WRITE(MIPI_DEVICE_READY(pipe), ULPS_STATE_ENTER); + I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY | ULPS_STATE_ENTER); usleep_range(2000, 2500); - I915_WRITE(MIPI_DEVICE_READY(pipe), ULPS_STATE_EXIT); + I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY | ULPS_STATE_EXIT); usleep_range(2000, 2500); - I915_WRITE(MIPI_DEVICE_READY(pipe), ULPS_STATE_ENTER); + I915_WRITE(MIPI_DEVICE_READY(pipe), DEVICE_READY | ULPS_STATE_ENTER); usleep_range(2000, 2500); - val = I915_READ(MIPI_PORT_CTRL(pipe)); - I915_WRITE(MIPI_PORT_CTRL(pipe), val & ~LP_OUTPUT_HOLD); - usleep_range(1000, 1500); - if (wait_for(((I915_READ(MIPI_PORT_CTRL(pipe)) & AFE_LATCHOUT) == 0x00000), 30)) DRM_ERROR("DSI LP not going Low\n"); + val = I915_READ(MIPI_PORT_CTRL(pipe)); + I915_WRITE(MIPI_PORT_CTRL(pipe), val & ~LP_OUTPUT_HOLD); + usleep_range(1000, 1500); + I915_WRITE(MIPI_DEVICE_READY(pipe), 0x00); usleep_range(2000, 2500); vlv_disable_dsi_pll(encoder); } + static void intel_dsi_post_disable(struct intel_encoder *encoder) { + struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + u32 val; DRM_DEBUG_KMS("\n"); + intel_dsi_disable(encoder); + intel_dsi_clear_device_ready(encoder); + val = I915_READ(DSPCLK_GATE_D); + val &= ~DPOUNIT_CLOCK_GATE_DISABLE; + I915_WRITE(DSPCLK_GATE_D, val); + if (intel_dsi->dev.dev_ops->disable_panel_power) intel_dsi->dev.dev_ops->disable_panel_power(&intel_dsi->dev); + + msleep(intel_dsi->panel_off_delay); + msleep(intel_dsi->panel_pwr_cycle_delay); } static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + enum intel_display_power_domain power_domain; u32 port, func; enum pipe p; DRM_DEBUG_KMS("\n"); + power_domain = intel_display_port_power_domain(encoder); + if (!intel_display_power_enabled(dev_priv, power_domain)) + return false; + /* XXX: this only works for one DSI output */ for (p = PIPE_A; p <= PIPE_B; p++) { port = I915_READ(MIPI_PORT_CTRL(p)); @@ -359,7 +443,7 @@ static void set_dsi_timings(struct drm_encoder *encoder, I915_WRITE(MIPI_VBP_COUNT(pipe), vbp); } -static void intel_dsi_mode_set(struct intel_encoder *intel_encoder) +static void intel_dsi_prepare(struct intel_encoder *intel_encoder) { struct drm_encoder *encoder = &intel_encoder->base; struct drm_device *dev = encoder->dev; @@ -374,9 +458,6 @@ static void intel_dsi_mode_set(struct intel_encoder *intel_encoder) DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe)); - /* XXX: Location of the call */ - band_gap_reset(dev_priv); - /* escape clock divider, 20MHz, shared for A and C. device ready must be * off when doing this! txclkesc? */ tmp = I915_READ(MIPI_CTRL(0)); @@ -447,10 +528,20 @@ static void intel_dsi_mode_set(struct intel_encoder *intel_encoder) /* dphy stuff */ /* in terms of low power clock */ - I915_WRITE(MIPI_INIT_COUNT(pipe), txclkesc(ESCAPE_CLOCK_DIVIDER_1, 100)); + I915_WRITE(MIPI_INIT_COUNT(pipe), txclkesc(intel_dsi->escape_clk_div, 100)); + + val = 0; + if (intel_dsi->eotp_pkt == 0) + val |= EOT_DISABLE; + + if (intel_dsi->clock_stop) + val |= CLOCKSTOP; /* recovery disables */ - I915_WRITE(MIPI_EOT_DISABLE(pipe), intel_dsi->eot_disable); + I915_WRITE(MIPI_EOT_DISABLE(pipe), val); + + /* in terms of low power clock */ + I915_WRITE(MIPI_INIT_COUNT(pipe), intel_dsi->init_count); /* in terms of txbyteclkhs. actual high to low switch + * MIPI_STOP_STATE_STALL * MIPI_LP_BYTECLK. @@ -479,17 +570,42 @@ static void intel_dsi_mode_set(struct intel_encoder *intel_encoder) intel_dsi->clk_hs_to_lp_count << HS_LP_PWR_SW_CNT_SHIFT); if (is_vid_mode(intel_dsi)) + /* Some panels might have resolution which is not a multiple of + * 64 like 1366 x 768. Enable RANDOM resolution support for such + * panels by default */ I915_WRITE(MIPI_VIDEO_MODE_FORMAT(pipe), intel_dsi->video_frmt_cfg_bits | - intel_dsi->video_mode_format); + intel_dsi->video_mode_format | + IP_TG_CONFIG | + RANDOM_DPI_DISPLAY_RESOLUTION); +} + +static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder) +{ + DRM_DEBUG_KMS("\n"); + + intel_dsi_prepare(encoder); + + vlv_enable_dsi_pll(encoder); } static enum drm_connector_status intel_dsi_detect(struct drm_connector *connector, bool force) { struct intel_dsi *intel_dsi = intel_attached_dsi(connector); + struct intel_encoder *intel_encoder = &intel_dsi->base; + enum intel_display_power_domain power_domain; + enum drm_connector_status connector_status; + struct drm_i915_private *dev_priv = intel_encoder->base.dev->dev_private; + DRM_DEBUG_KMS("\n"); - return intel_dsi->dev.dev_ops->detect(&intel_dsi->dev); + power_domain = intel_display_port_power_domain(intel_encoder); + + intel_display_power_get(dev_priv, power_domain); + connector_status = intel_dsi->dev.dev_ops->detect(&intel_dsi->dev); + intel_display_power_put(dev_priv, power_domain); + + return connector_status; } static int intel_dsi_get_modes(struct drm_connector *connector) @@ -550,11 +666,16 @@ bool intel_dsi_init(struct drm_device *dev) struct intel_connector *intel_connector; struct drm_connector *connector; struct drm_display_mode *fixed_mode = NULL; + struct drm_i915_private *dev_priv = dev->dev_private; const struct intel_dsi_device *dsi; unsigned int i; DRM_DEBUG_KMS("\n"); + /* There is no detection method for MIPI so rely on VBT */ + if (!dev_priv->vbt.has_mipi) + return false; + intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL); if (!intel_dsi) return false; @@ -569,6 +690,13 @@ bool intel_dsi_init(struct drm_device *dev) encoder = &intel_encoder->base; intel_dsi->attached_connector = intel_connector; + if (IS_VALLEYVIEW(dev)) { + dev_priv->mipi_mmio_base = VLV_MIPI_BASE; + } else { + DRM_ERROR("Unsupported Mipi device to reg base"); + return false; + } + connector = &intel_connector->base; drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI); @@ -578,14 +706,14 @@ bool intel_dsi_init(struct drm_device *dev) intel_encoder->compute_config = intel_dsi_compute_config; intel_encoder->pre_pll_enable = intel_dsi_pre_pll_enable; intel_encoder->pre_enable = intel_dsi_pre_enable; - intel_encoder->enable = intel_dsi_enable; - intel_encoder->mode_set = intel_dsi_mode_set; - intel_encoder->disable = intel_dsi_disable; + intel_encoder->enable = intel_dsi_enable_nop; + intel_encoder->disable = intel_dsi_pre_disable; intel_encoder->post_disable = intel_dsi_post_disable; intel_encoder->get_hw_state = intel_dsi_get_hw_state; intel_encoder->get_config = intel_dsi_get_config; intel_connector->get_hw_state = intel_connector_get_hw_state; + intel_connector->unregister = intel_connector_unregister; for (i = 0; i < ARRAY_SIZE(intel_dsi_devices); i++) { dsi = &intel_dsi_devices[i]; @@ -603,7 +731,7 @@ bool intel_dsi_init(struct drm_device *dev) intel_encoder->type = INTEL_OUTPUT_DSI; intel_encoder->crtc_mask = (1 << 0); /* XXX */ - intel_encoder->cloneable = false; + intel_encoder->cloneable = 0; drm_connector_init(dev, connector, &intel_dsi_connector_funcs, DRM_MODE_CONNECTOR_DSI); @@ -624,7 +752,7 @@ bool intel_dsi_init(struct drm_device *dev) } fixed_mode->type |= DRM_MODE_TYPE_PREFERRED; - intel_panel_init(&intel_connector->panel, fixed_mode); + intel_panel_init(&intel_connector->panel, fixed_mode, NULL); return true; diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h index b4a27cec882..31db33d3e5c 100644 --- a/drivers/gpu/drm/i915/intel_dsi.h +++ b/drivers/gpu/drm/i915/intel_dsi.h @@ -31,7 +31,6 @@ struct intel_dsi_device { unsigned int panel_id; const char *name; - int type; const struct intel_dsi_dev_ops *dev_ops; void *dev_priv; }; @@ -85,6 +84,9 @@ struct intel_dsi { /* virtual channel */ int channel; + /* Video mode or command mode */ + u16 operation_mode; + /* number of DSI lanes */ unsigned int lane_count; @@ -95,8 +97,10 @@ struct intel_dsi { u32 video_mode_format; /* eot for MIPI_EOT_DISABLE register */ - u32 eot_disable; + u8 eotp_pkt; + u8 clock_stop; + u8 escape_clk_div; u32 port_bits; u32 bw_timer; u32 dphy_reg; @@ -110,6 +114,15 @@ struct intel_dsi { u16 hs_to_lp_count; u16 clk_lp_to_hs_count; u16 clk_hs_to_lp_count; + + u16 init_count; + + /* all delays in ms */ + u16 backlight_off_delay; + u16 backlight_on_delay; + u16 panel_on_delay; + u16 panel_off_delay; + u16 panel_pwr_cycle_delay; }; static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder) @@ -120,4 +133,6 @@ static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder) extern void vlv_enable_dsi_pll(struct intel_encoder *encoder); extern void vlv_disable_dsi_pll(struct intel_encoder *encoder); +extern struct intel_dsi_dev_ops vbt_generic_dsi_display_ops; + #endif /* _INTEL_DSI_H */ diff --git a/drivers/gpu/drm/i915/intel_dsi_cmd.c b/drivers/gpu/drm/i915/intel_dsi_cmd.c index 7c40f981d2c..933c8630523 100644 --- a/drivers/gpu/drm/i915/intel_dsi_cmd.c +++ b/drivers/gpu/drm/i915/intel_dsi_cmd.c @@ -389,7 +389,7 @@ int dsi_vc_generic_read(struct intel_dsi *intel_dsi, int channel, * * XXX: commands with data in MIPI_DPI_DATA? */ -int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd) +int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs) { struct drm_encoder *encoder = &intel_dsi->base.base; struct drm_device *dev = encoder->dev; @@ -399,17 +399,11 @@ int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd) u32 mask; /* XXX: pipe, hs */ - if (intel_dsi->hs) + if (hs) cmd &= ~DPI_LP_MODE; else cmd |= DPI_LP_MODE; - /* DPI virtual channel?! */ - - mask = DPI_FIFO_EMPTY; - if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(pipe)) & mask) == mask, 50)) - DRM_ERROR("Timeout waiting for DPI FIFO empty.\n"); - /* clear bit */ I915_WRITE(MIPI_INTR_STAT(pipe), SPL_PKT_SENT_INTERRUPT); diff --git a/drivers/gpu/drm/i915/intel_dsi_cmd.h b/drivers/gpu/drm/i915/intel_dsi_cmd.h index 54c8a234a2e..9a18cbfa546 100644 --- a/drivers/gpu/drm/i915/intel_dsi_cmd.h +++ b/drivers/gpu/drm/i915/intel_dsi_cmd.h @@ -33,6 +33,9 @@ #include "intel_drv.h" #include "intel_dsi.h" +#define DPI_LP_MODE_EN false +#define DPI_HS_MODE_EN true + void dsi_hs_mode_enable(struct intel_dsi *intel_dsi, bool enable); int dsi_vc_dcs_write(struct intel_dsi *intel_dsi, int channel, @@ -47,7 +50,7 @@ int dsi_vc_dcs_read(struct intel_dsi *intel_dsi, int channel, u8 dcs_cmd, int dsi_vc_generic_read(struct intel_dsi *intel_dsi, int channel, u8 *reqdata, int reqlen, u8 *buf, int buflen); -int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd); +int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs); /* XXX: questionable write helpers */ static inline int dsi_vc_dcs_write_0(struct intel_dsi *intel_dsi, diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c new file mode 100644 index 00000000000..21a0d348ced --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c @@ -0,0 +1,589 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Shobhit Kumar <shobhit.kumar@intel.com> + * + */ + +#include <drm/drmP.h> +#include <drm/drm_crtc.h> +#include <drm/drm_edid.h> +#include <drm/i915_drm.h> +#include <linux/slab.h> +#include <video/mipi_display.h> +#include <asm/intel-mid.h> +#include <video/mipi_display.h> +#include "i915_drv.h" +#include "intel_drv.h" +#include "intel_dsi.h" +#include "intel_dsi_cmd.h" + +#define MIPI_TRANSFER_MODE_SHIFT 0 +#define MIPI_VIRTUAL_CHANNEL_SHIFT 1 +#define MIPI_PORT_SHIFT 3 + +#define PREPARE_CNT_MAX 0x3F +#define EXIT_ZERO_CNT_MAX 0x3F +#define CLK_ZERO_CNT_MAX 0xFF +#define TRAIL_CNT_MAX 0x1F + +#define NS_KHZ_RATIO 1000000 + +#define GPI0_NC_0_HV_DDI0_HPD 0x4130 +#define GPIO_NC_0_HV_DDI0_PAD 0x4138 +#define GPIO_NC_1_HV_DDI0_DDC_SDA 0x4120 +#define GPIO_NC_1_HV_DDI0_DDC_SDA_PAD 0x4128 +#define GPIO_NC_2_HV_DDI0_DDC_SCL 0x4110 +#define GPIO_NC_2_HV_DDI0_DDC_SCL_PAD 0x4118 +#define GPIO_NC_3_PANEL0_VDDEN 0x4140 +#define GPIO_NC_3_PANEL0_VDDEN_PAD 0x4148 +#define GPIO_NC_4_PANEL0_BLKEN 0x4150 +#define GPIO_NC_4_PANEL0_BLKEN_PAD 0x4158 +#define GPIO_NC_5_PANEL0_BLKCTL 0x4160 +#define GPIO_NC_5_PANEL0_BLKCTL_PAD 0x4168 +#define GPIO_NC_6_PCONF0 0x4180 +#define GPIO_NC_6_PAD 0x4188 +#define GPIO_NC_7_PCONF0 0x4190 +#define GPIO_NC_7_PAD 0x4198 +#define GPIO_NC_8_PCONF0 0x4170 +#define GPIO_NC_8_PAD 0x4178 +#define GPIO_NC_9_PCONF0 0x4100 +#define GPIO_NC_9_PAD 0x4108 +#define GPIO_NC_10_PCONF0 0x40E0 +#define GPIO_NC_10_PAD 0x40E8 +#define GPIO_NC_11_PCONF0 0x40F0 +#define GPIO_NC_11_PAD 0x40F8 + +struct gpio_table { + u16 function_reg; + u16 pad_reg; + u8 init; +}; + +static struct gpio_table gtable[] = { + { GPI0_NC_0_HV_DDI0_HPD, GPIO_NC_0_HV_DDI0_PAD, 0 }, + { GPIO_NC_1_HV_DDI0_DDC_SDA, GPIO_NC_1_HV_DDI0_DDC_SDA_PAD, 0 }, + { GPIO_NC_2_HV_DDI0_DDC_SCL, GPIO_NC_2_HV_DDI0_DDC_SCL_PAD, 0 }, + { GPIO_NC_3_PANEL0_VDDEN, GPIO_NC_3_PANEL0_VDDEN_PAD, 0 }, + { GPIO_NC_4_PANEL0_BLKEN, GPIO_NC_4_PANEL0_BLKEN_PAD, 0 }, + { GPIO_NC_5_PANEL0_BLKCTL, GPIO_NC_5_PANEL0_BLKCTL_PAD, 0 }, + { GPIO_NC_6_PCONF0, GPIO_NC_6_PAD, 0 }, + { GPIO_NC_7_PCONF0, GPIO_NC_7_PAD, 0 }, + { GPIO_NC_8_PCONF0, GPIO_NC_8_PAD, 0 }, + { GPIO_NC_9_PCONF0, GPIO_NC_9_PAD, 0 }, + { GPIO_NC_10_PCONF0, GPIO_NC_10_PAD, 0}, + { GPIO_NC_11_PCONF0, GPIO_NC_11_PAD, 0} +}; + +static u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi, u8 *data) +{ + u8 type, byte, mode, vc, port; + u16 len; + + byte = *data++; + mode = (byte >> MIPI_TRANSFER_MODE_SHIFT) & 0x1; + vc = (byte >> MIPI_VIRTUAL_CHANNEL_SHIFT) & 0x3; + port = (byte >> MIPI_PORT_SHIFT) & 0x3; + + /* LP or HS mode */ + intel_dsi->hs = mode; + + /* get packet type and increment the pointer */ + type = *data++; + + len = *((u16 *) data); + data += 2; + + switch (type) { + case MIPI_DSI_GENERIC_SHORT_WRITE_0_PARAM: + dsi_vc_generic_write_0(intel_dsi, vc); + break; + case MIPI_DSI_GENERIC_SHORT_WRITE_1_PARAM: + dsi_vc_generic_write_1(intel_dsi, vc, *data); + break; + case MIPI_DSI_GENERIC_SHORT_WRITE_2_PARAM: + dsi_vc_generic_write_2(intel_dsi, vc, *data, *(data + 1)); + break; + case MIPI_DSI_GENERIC_READ_REQUEST_0_PARAM: + case MIPI_DSI_GENERIC_READ_REQUEST_1_PARAM: + case MIPI_DSI_GENERIC_READ_REQUEST_2_PARAM: + DRM_DEBUG_DRIVER("Generic Read not yet implemented or used\n"); + break; + case MIPI_DSI_GENERIC_LONG_WRITE: + dsi_vc_generic_write(intel_dsi, vc, data, len); + break; + case MIPI_DSI_DCS_SHORT_WRITE: + dsi_vc_dcs_write_0(intel_dsi, vc, *data); + break; + case MIPI_DSI_DCS_SHORT_WRITE_PARAM: + dsi_vc_dcs_write_1(intel_dsi, vc, *data, *(data + 1)); + break; + case MIPI_DSI_DCS_READ: + DRM_DEBUG_DRIVER("DCS Read not yet implemented or used\n"); + break; + case MIPI_DSI_DCS_LONG_WRITE: + dsi_vc_dcs_write(intel_dsi, vc, data, len); + break; + }; + + data += len; + + return data; +} + +static u8 *mipi_exec_delay(struct intel_dsi *intel_dsi, u8 *data) +{ + u32 delay = *((u32 *) data); + + usleep_range(delay, delay + 10); + data += 4; + + return data; +} + +static u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, u8 *data) +{ + u8 gpio, action; + u16 function, pad; + u32 val; + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + gpio = *data++; + + /* pull up/down */ + action = *data++; + + function = gtable[gpio].function_reg; + pad = gtable[gpio].pad_reg; + + mutex_lock(&dev_priv->dpio_lock); + if (!gtable[gpio].init) { + /* program the function */ + /* FIXME: remove constant below */ + vlv_gpio_nc_write(dev_priv, function, 0x2000CC00); + gtable[gpio].init = 1; + } + + val = 0x4 | action; + + /* pull up/down */ + vlv_gpio_nc_write(dev_priv, pad, val); + mutex_unlock(&dev_priv->dpio_lock); + + return data; +} + +typedef u8 * (*fn_mipi_elem_exec)(struct intel_dsi *intel_dsi, u8 *data); +static const fn_mipi_elem_exec exec_elem[] = { + NULL, /* reserved */ + mipi_exec_send_packet, + mipi_exec_delay, + mipi_exec_gpio, + NULL, /* status read; later */ +}; + +/* + * MIPI Sequence from VBT #53 parsing logic + * We have already separated each seqence during bios parsing + * Following is generic execution function for any sequence + */ + +static const char * const seq_name[] = { + "UNDEFINED", + "MIPI_SEQ_ASSERT_RESET", + "MIPI_SEQ_INIT_OTP", + "MIPI_SEQ_DISPLAY_ON", + "MIPI_SEQ_DISPLAY_OFF", + "MIPI_SEQ_DEASSERT_RESET" +}; + +static void generic_exec_sequence(struct intel_dsi *intel_dsi, char *sequence) +{ + u8 *data = sequence; + fn_mipi_elem_exec mipi_elem_exec; + int index; + + if (!sequence) + return; + + DRM_DEBUG_DRIVER("Starting MIPI sequence - %s\n", seq_name[*data]); + + /* go to the first element of the sequence */ + data++; + + /* parse each byte till we reach end of sequence byte - 0x00 */ + while (1) { + index = *data; + mipi_elem_exec = exec_elem[index]; + if (!mipi_elem_exec) { + DRM_ERROR("Unsupported MIPI element, skipping sequence execution\n"); + return; + } + + /* goto element payload */ + data++; + + /* execute the element specific rotines */ + data = mipi_elem_exec(intel_dsi, data); + + /* + * After processing the element, data should point to + * next element or end of sequence + * check if have we reached end of sequence + */ + if (*data == 0x00) + break; + } +} + +static bool generic_init(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct mipi_config *mipi_config = dev_priv->vbt.dsi.config; + struct mipi_pps_data *pps = dev_priv->vbt.dsi.pps; + struct drm_display_mode *mode = dev_priv->vbt.lfp_lvds_vbt_mode; + u32 bits_per_pixel = 24; + u32 tlpx_ns, extra_byte_count, bitrate, tlpx_ui; + u32 ui_num, ui_den; + u32 prepare_cnt, exit_zero_cnt, clk_zero_cnt, trail_cnt; + u32 ths_prepare_ns, tclk_trail_ns; + u32 tclk_prepare_clkzero, ths_prepare_hszero; + u32 lp_to_hs_switch, hs_to_lp_switch; + + DRM_DEBUG_KMS("\n"); + + intel_dsi->eotp_pkt = mipi_config->eot_pkt_disabled ? 0 : 1; + intel_dsi->clock_stop = mipi_config->enable_clk_stop ? 1 : 0; + intel_dsi->lane_count = mipi_config->lane_cnt + 1; + intel_dsi->pixel_format = mipi_config->videomode_color_format << 7; + + if (intel_dsi->pixel_format == VID_MODE_FORMAT_RGB666) + bits_per_pixel = 18; + else if (intel_dsi->pixel_format == VID_MODE_FORMAT_RGB565) + bits_per_pixel = 16; + + bitrate = (mode->clock * bits_per_pixel) / intel_dsi->lane_count; + + intel_dsi->operation_mode = mipi_config->is_cmd_mode; + intel_dsi->video_mode_format = mipi_config->video_transfer_mode; + intel_dsi->escape_clk_div = mipi_config->byte_clk_sel; + intel_dsi->lp_rx_timeout = mipi_config->lp_rx_timeout; + intel_dsi->turn_arnd_val = mipi_config->turn_around_timeout; + intel_dsi->rst_timer_val = mipi_config->device_reset_timer; + intel_dsi->init_count = mipi_config->master_init_timer; + intel_dsi->bw_timer = mipi_config->dbi_bw_timer; + intel_dsi->video_frmt_cfg_bits = mipi_config->bta_enabled ? DISABLE_VIDEO_BTA : 0; + + switch (intel_dsi->escape_clk_div) { + case 0: + tlpx_ns = 50; + break; + case 1: + tlpx_ns = 100; + break; + + case 2: + tlpx_ns = 200; + break; + default: + tlpx_ns = 50; + break; + } + + switch (intel_dsi->lane_count) { + case 1: + case 2: + extra_byte_count = 2; + break; + case 3: + extra_byte_count = 4; + break; + case 4: + default: + extra_byte_count = 3; + break; + } + + /* + * ui(s) = 1/f [f in hz] + * ui(ns) = 10^9 / (f*10^6) [f in Mhz] -> 10^3/f(Mhz) + */ + + /* in Kbps */ + ui_num = NS_KHZ_RATIO; + ui_den = bitrate; + + tclk_prepare_clkzero = mipi_config->tclk_prepare_clkzero; + ths_prepare_hszero = mipi_config->ths_prepare_hszero; + + /* + * B060 + * LP byte clock = TLPX/ (8UI) + */ + intel_dsi->lp_byte_clk = DIV_ROUND_UP(tlpx_ns * ui_den, 8 * ui_num); + + /* count values in UI = (ns value) * (bitrate / (2 * 10^6)) + * + * Since txddrclkhs_i is 2xUI, all the count values programmed in + * DPHY param register are divided by 2 + * + * prepare count + */ + ths_prepare_ns = max(mipi_config->ths_prepare, mipi_config->tclk_prepare); + prepare_cnt = DIV_ROUND_UP(ths_prepare_ns * ui_den, ui_num * 2); + + /* exit zero count */ + exit_zero_cnt = DIV_ROUND_UP( + (ths_prepare_hszero - ths_prepare_ns) * ui_den, + ui_num * 2 + ); + + /* + * Exit zero is unified val ths_zero and ths_exit + * minimum value for ths_exit = 110ns + * min (exit_zero_cnt * 2) = 110/UI + * exit_zero_cnt = 55/UI + */ + if (exit_zero_cnt < (55 * ui_den / ui_num)) + if ((55 * ui_den) % ui_num) + exit_zero_cnt += 1; + + /* clk zero count */ + clk_zero_cnt = DIV_ROUND_UP( + (tclk_prepare_clkzero - ths_prepare_ns) + * ui_den, 2 * ui_num); + + /* trail count */ + tclk_trail_ns = max(mipi_config->tclk_trail, mipi_config->ths_trail); + trail_cnt = DIV_ROUND_UP(tclk_trail_ns * ui_den, 2 * ui_num); + + if (prepare_cnt > PREPARE_CNT_MAX || + exit_zero_cnt > EXIT_ZERO_CNT_MAX || + clk_zero_cnt > CLK_ZERO_CNT_MAX || + trail_cnt > TRAIL_CNT_MAX) + DRM_DEBUG_DRIVER("Values crossing maximum limits, restricting to max values\n"); + + if (prepare_cnt > PREPARE_CNT_MAX) + prepare_cnt = PREPARE_CNT_MAX; + + if (exit_zero_cnt > EXIT_ZERO_CNT_MAX) + exit_zero_cnt = EXIT_ZERO_CNT_MAX; + + if (clk_zero_cnt > CLK_ZERO_CNT_MAX) + clk_zero_cnt = CLK_ZERO_CNT_MAX; + + if (trail_cnt > TRAIL_CNT_MAX) + trail_cnt = TRAIL_CNT_MAX; + + /* B080 */ + intel_dsi->dphy_reg = exit_zero_cnt << 24 | trail_cnt << 16 | + clk_zero_cnt << 8 | prepare_cnt; + + /* + * LP to HS switch count = 4TLPX + PREP_COUNT * 2 + EXIT_ZERO_COUNT * 2 + * + 10UI + Extra Byte Count + * + * HS to LP switch count = THS-TRAIL + 2TLPX + Extra Byte Count + * Extra Byte Count is calculated according to number of lanes. + * High Low Switch Count is the Max of LP to HS and + * HS to LP switch count + * + */ + tlpx_ui = DIV_ROUND_UP(tlpx_ns * ui_den, ui_num); + + /* B044 */ + /* FIXME: + * The comment above does not match with the code */ + lp_to_hs_switch = DIV_ROUND_UP(4 * tlpx_ui + prepare_cnt * 2 + + exit_zero_cnt * 2 + 10, 8); + + hs_to_lp_switch = DIV_ROUND_UP(mipi_config->ths_trail + 2 * tlpx_ui, 8); + + intel_dsi->hs_to_lp_count = max(lp_to_hs_switch, hs_to_lp_switch); + intel_dsi->hs_to_lp_count += extra_byte_count; + + /* B088 */ + /* LP -> HS for clock lanes + * LP clk sync + LP11 + LP01 + tclk_prepare + tclk_zero + + * extra byte count + * 2TPLX + 1TLPX + 1 TPLX(in ns) + prepare_cnt * 2 + clk_zero_cnt * + * 2(in UI) + extra byte count + * In byteclks = (4TLPX + prepare_cnt * 2 + clk_zero_cnt *2 (in UI)) / + * 8 + extra byte count + */ + intel_dsi->clk_lp_to_hs_count = + DIV_ROUND_UP( + 4 * tlpx_ui + prepare_cnt * 2 + + clk_zero_cnt * 2, + 8); + + intel_dsi->clk_lp_to_hs_count += extra_byte_count; + + /* HS->LP for Clock Lanes + * Low Power clock synchronisations + 1Tx byteclk + tclk_trail + + * Extra byte count + * 2TLPX + 8UI + (trail_count*2)(in UI) + Extra byte count + * In byteclks = (2*TLpx(in UI) + trail_count*2 +8)(in UI)/8 + + * Extra byte count + */ + intel_dsi->clk_hs_to_lp_count = + DIV_ROUND_UP(2 * tlpx_ui + trail_cnt * 2 + 8, + 8); + intel_dsi->clk_hs_to_lp_count += extra_byte_count; + + DRM_DEBUG_KMS("Eot %s\n", intel_dsi->eotp_pkt ? "enabled" : "disabled"); + DRM_DEBUG_KMS("Clockstop %s\n", intel_dsi->clock_stop ? + "disabled" : "enabled"); + DRM_DEBUG_KMS("Mode %s\n", intel_dsi->operation_mode ? "command" : "video"); + DRM_DEBUG_KMS("Pixel Format %d\n", intel_dsi->pixel_format); + DRM_DEBUG_KMS("TLPX %d\n", intel_dsi->escape_clk_div); + DRM_DEBUG_KMS("LP RX Timeout 0x%x\n", intel_dsi->lp_rx_timeout); + DRM_DEBUG_KMS("Turnaround Timeout 0x%x\n", intel_dsi->turn_arnd_val); + DRM_DEBUG_KMS("Init Count 0x%x\n", intel_dsi->init_count); + DRM_DEBUG_KMS("HS to LP Count 0x%x\n", intel_dsi->hs_to_lp_count); + DRM_DEBUG_KMS("LP Byte Clock %d\n", intel_dsi->lp_byte_clk); + DRM_DEBUG_KMS("DBI BW Timer 0x%x\n", intel_dsi->bw_timer); + DRM_DEBUG_KMS("LP to HS Clock Count 0x%x\n", intel_dsi->clk_lp_to_hs_count); + DRM_DEBUG_KMS("HS to LP Clock Count 0x%x\n", intel_dsi->clk_hs_to_lp_count); + DRM_DEBUG_KMS("BTA %s\n", + intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA ? + "disabled" : "enabled"); + + /* delays in VBT are in unit of 100us, so need to convert + * here in ms + * Delay (100us) * 100 /1000 = Delay / 10 (ms) */ + intel_dsi->backlight_off_delay = pps->bl_disable_delay / 10; + intel_dsi->backlight_on_delay = pps->bl_enable_delay / 10; + intel_dsi->panel_on_delay = pps->panel_on_delay / 10; + intel_dsi->panel_off_delay = pps->panel_off_delay / 10; + intel_dsi->panel_pwr_cycle_delay = pps->panel_power_cycle_delay / 10; + + return true; +} + +static int generic_mode_valid(struct intel_dsi_device *dsi, + struct drm_display_mode *mode) +{ + return MODE_OK; +} + +static bool generic_mode_fixup(struct intel_dsi_device *dsi, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) { + return true; +} + +static void generic_panel_reset(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + char *sequence = dev_priv->vbt.dsi.sequence[MIPI_SEQ_ASSERT_RESET]; + + generic_exec_sequence(intel_dsi, sequence); +} + +static void generic_disable_panel_power(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + char *sequence = dev_priv->vbt.dsi.sequence[MIPI_SEQ_DEASSERT_RESET]; + + generic_exec_sequence(intel_dsi, sequence); +} + +static void generic_send_otp_cmds(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + char *sequence = dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP]; + + generic_exec_sequence(intel_dsi, sequence); +} + +static void generic_enable(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + char *sequence = dev_priv->vbt.dsi.sequence[MIPI_SEQ_DISPLAY_ON]; + + generic_exec_sequence(intel_dsi, sequence); +} + +static void generic_disable(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + char *sequence = dev_priv->vbt.dsi.sequence[MIPI_SEQ_DISPLAY_OFF]; + + generic_exec_sequence(intel_dsi, sequence); +} + +static enum drm_connector_status generic_detect(struct intel_dsi_device *dsi) +{ + return connector_status_connected; +} + +static bool generic_get_hw_state(struct intel_dsi_device *dev) +{ + return true; +} + +static struct drm_display_mode *generic_get_modes(struct intel_dsi_device *dsi) +{ + struct intel_dsi *intel_dsi = container_of(dsi, struct intel_dsi, dev); + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + dev_priv->vbt.lfp_lvds_vbt_mode->type |= DRM_MODE_TYPE_PREFERRED; + return dev_priv->vbt.lfp_lvds_vbt_mode; +} + +static void generic_destroy(struct intel_dsi_device *dsi) { } + +/* Callbacks. We might not need them all. */ +struct intel_dsi_dev_ops vbt_generic_dsi_display_ops = { + .init = generic_init, + .mode_valid = generic_mode_valid, + .mode_fixup = generic_mode_fixup, + .panel_reset = generic_panel_reset, + .disable_panel_power = generic_disable_panel_power, + .send_otp_cmds = generic_send_otp_cmds, + .enable = generic_enable, + .disable = generic_disable, + .detect = generic_detect, + .get_hw_state = generic_get_hw_state, + .get_modes = generic_get_modes, + .destroy = generic_destroy, +}; diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index eeff998e52e..a3631c0a5c2 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -285,7 +285,7 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder, return true; } -static void intel_dvo_mode_set(struct intel_encoder *encoder) +static void intel_dvo_pre_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -343,7 +343,7 @@ intel_dvo_detect(struct drm_connector *connector, bool force) { struct intel_dvo *intel_dvo = intel_attached_dvo(connector); DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); return intel_dvo->dev.dev_ops->detect(&intel_dvo->dev); } @@ -475,8 +475,9 @@ void intel_dvo_init(struct drm_device *dev) intel_encoder->get_hw_state = intel_dvo_get_hw_state; intel_encoder->get_config = intel_dvo_get_config; intel_encoder->compute_config = intel_dvo_compute_config; - intel_encoder->mode_set = intel_dvo_mode_set; + intel_encoder->pre_enable = intel_dvo_pre_enable; intel_connector->get_hw_state = intel_dvo_connector_get_hw_state; + intel_connector->unregister = intel_connector_unregister; /* Now, try to find a controller */ for (i = 0; i < ARRAY_SIZE(intel_dvo_devices); i++) { @@ -521,14 +522,15 @@ void intel_dvo_init(struct drm_device *dev) intel_encoder->crtc_mask = (1 << 0) | (1 << 1); switch (dvo->type) { case INTEL_DVO_CHIP_TMDS: - intel_encoder->cloneable = true; + intel_encoder->cloneable = (1 << INTEL_OUTPUT_ANALOG) | + (1 << INTEL_OUTPUT_DVO); drm_connector_init(dev, connector, &intel_dvo_connector_funcs, DRM_MODE_CONNECTOR_DVII); encoder_type = DRM_MODE_ENCODER_TMDS; break; case INTEL_DVO_CHIP_LVDS: - intel_encoder->cloneable = false; + intel_encoder->cloneable = 0; drm_connector_init(dev, connector, &intel_dvo_connector_funcs, DRM_MODE_CONNECTOR_LVDS); diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index 39eac9937a4..088fe9378a4 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -62,6 +62,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper, { struct intel_fbdev *ifbdev = container_of(helper, struct intel_fbdev, helper); + struct drm_framebuffer *fb; struct drm_device *dev = helper->dev; struct drm_mode_fb_cmd2 mode_cmd = {}; struct drm_i915_gem_object *obj; @@ -93,18 +94,22 @@ static int intelfb_alloc(struct drm_fb_helper *helper, /* Flush everything out, we'll be doing GTT only from now on */ ret = intel_pin_and_fence_fb_obj(dev, obj, NULL); if (ret) { - DRM_ERROR("failed to pin fb: %d\n", ret); + DRM_ERROR("failed to pin obj: %d\n", ret); goto out_unref; } - ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj); - if (ret) + fb = __intel_framebuffer_create(dev, &mode_cmd, obj); + if (IS_ERR(fb)) { + ret = PTR_ERR(fb); goto out_unpin; + } + + ifbdev->fb = to_intel_framebuffer(fb); return 0; out_unpin: - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); out_unref: drm_gem_object_unreference(&obj->base); out: @@ -116,23 +121,36 @@ static int intelfb_create(struct drm_fb_helper *helper, { struct intel_fbdev *ifbdev = container_of(helper, struct intel_fbdev, helper); - struct intel_framebuffer *intel_fb = &ifbdev->ifb; + struct intel_framebuffer *intel_fb = ifbdev->fb; struct drm_device *dev = helper->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct fb_info *info; struct drm_framebuffer *fb; struct drm_i915_gem_object *obj; int size, ret; + bool prealloc = false; mutex_lock(&dev->struct_mutex); - if (!intel_fb->obj) { + if (intel_fb && + (sizes->fb_width > intel_fb->base.width || + sizes->fb_height > intel_fb->base.height)) { + DRM_DEBUG_KMS("BIOS fb too small (%dx%d), we require (%dx%d)," + " releasing it\n", + intel_fb->base.width, intel_fb->base.height, + sizes->fb_width, sizes->fb_height); + drm_framebuffer_unreference(&intel_fb->base); + intel_fb = ifbdev->fb = NULL; + } + if (!intel_fb || WARN_ON(!intel_fb->obj)) { DRM_DEBUG_KMS("no BIOS fb, allocating a new one\n"); ret = intelfb_alloc(helper, sizes); if (ret) goto out_unlock; + intel_fb = ifbdev->fb; } else { DRM_DEBUG_KMS("re-using BIOS fb\n"); + prealloc = true; sizes->fb_width = intel_fb->base.width; sizes->fb_height = intel_fb->base.height; } @@ -148,7 +166,7 @@ static int intelfb_create(struct drm_fb_helper *helper, info->par = helper; - fb = &ifbdev->ifb.base; + fb = &ifbdev->fb->base; ifbdev->helper.fb = fb; ifbdev->helper.fbdev = info; @@ -194,7 +212,7 @@ static int intelfb_create(struct drm_fb_helper *helper, * If the object is stolen however, it will be full of whatever * garbage was left in there. */ - if (ifbdev->ifb.obj->stolen) + if (ifbdev->fb->obj->stolen && !prealloc) memset_io(info->screen_base, 0, info->screen_size); /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ @@ -208,7 +226,7 @@ static int intelfb_create(struct drm_fb_helper *helper, return 0; out_unpin: - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); drm_gem_object_unreference(&obj->base); out_unlock: mutex_unlock(&dev->struct_mutex); @@ -236,7 +254,206 @@ static void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green, *blue = intel_crtc->lut_b[regno] << 8; } +static struct drm_fb_helper_crtc * +intel_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc) +{ + int i; + + for (i = 0; i < fb_helper->crtc_count; i++) + if (fb_helper->crtc_info[i].mode_set.crtc == crtc) + return &fb_helper->crtc_info[i]; + + return NULL; +} + +/* + * Try to read the BIOS display configuration and use it for the initial + * fb configuration. + * + * The BIOS or boot loader will generally create an initial display + * configuration for us that includes some set of active pipes and displays. + * This routine tries to figure out which pipes and connectors are active + * and stuffs them into the crtcs and modes array given to us by the + * drm_fb_helper code. + * + * The overall sequence is: + * intel_fbdev_init - from driver load + * intel_fbdev_init_bios - initialize the intel_fbdev using BIOS data + * drm_fb_helper_init - build fb helper structs + * drm_fb_helper_single_add_all_connectors - more fb helper structs + * intel_fbdev_initial_config - apply the config + * drm_fb_helper_initial_config - call ->probe then register_framebuffer() + * drm_setup_crtcs - build crtc config for fbdev + * intel_fb_initial_config - find active connectors etc + * drm_fb_helper_single_fb_probe - set up fbdev + * intelfb_create - re-use or alloc fb, build out fbdev structs + * + * Note that we don't make special consideration whether we could actually + * switch to the selected modes without a full modeset. E.g. when the display + * is in VGA mode we need to recalculate watermarks and set a new high-res + * framebuffer anyway. + */ +static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, + struct drm_fb_helper_crtc **crtcs, + struct drm_display_mode **modes, + bool *enabled, int width, int height) +{ + struct drm_device *dev = fb_helper->dev; + int i, j; + bool *save_enabled; + bool fallback = true; + int num_connectors_enabled = 0; + int num_connectors_detected = 0; + + /* + * If the user specified any force options, just bail here + * and use that config. + */ + for (i = 0; i < fb_helper->connector_count; i++) { + struct drm_fb_helper_connector *fb_conn; + struct drm_connector *connector; + + fb_conn = fb_helper->connector_info[i]; + connector = fb_conn->connector; + + if (!enabled[i]) + continue; + + if (connector->force != DRM_FORCE_UNSPECIFIED) + return false; + } + + save_enabled = kcalloc(dev->mode_config.num_connector, sizeof(bool), + GFP_KERNEL); + if (!save_enabled) + return false; + + memcpy(save_enabled, enabled, dev->mode_config.num_connector); + + for (i = 0; i < fb_helper->connector_count; i++) { + struct drm_fb_helper_connector *fb_conn; + struct drm_connector *connector; + struct drm_encoder *encoder; + struct drm_fb_helper_crtc *new_crtc; + + fb_conn = fb_helper->connector_info[i]; + connector = fb_conn->connector; + + if (connector->status == connector_status_connected) + num_connectors_detected++; + + if (!enabled[i]) { + DRM_DEBUG_KMS("connector %s not enabled, skipping\n", + connector->name); + continue; + } + + encoder = connector->encoder; + if (!encoder || WARN_ON(!encoder->crtc)) { + DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n", + connector->name); + enabled[i] = false; + continue; + } + + num_connectors_enabled++; + + new_crtc = intel_fb_helper_crtc(fb_helper, encoder->crtc); + + /* + * Make sure we're not trying to drive multiple connectors + * with a single CRTC, since our cloning support may not + * match the BIOS. + */ + for (j = 0; j < fb_helper->connector_count; j++) { + if (crtcs[j] == new_crtc) { + DRM_DEBUG_KMS("fallback: cloned configuration\n"); + fallback = true; + goto out; + } + } + + DRM_DEBUG_KMS("looking for cmdline mode on connector %s\n", + connector->name); + + /* go for command line mode first */ + modes[i] = drm_pick_cmdline_mode(fb_conn, width, height); + + /* try for preferred next */ + if (!modes[i]) { + DRM_DEBUG_KMS("looking for preferred mode on connector %s\n", + connector->name); + modes[i] = drm_has_preferred_mode(fb_conn, width, + height); + } + + /* No preferred mode marked by the EDID? Are there any modes? */ + if (!modes[i] && !list_empty(&connector->modes)) { + DRM_DEBUG_KMS("using first mode listed on connector %s\n", + connector->name); + modes[i] = list_first_entry(&connector->modes, + struct drm_display_mode, + head); + } + + /* last resort: use current mode */ + if (!modes[i]) { + /* + * IMPORTANT: We want to use the adjusted mode (i.e. + * after the panel fitter upscaling) as the initial + * config, not the input mode, which is what crtc->mode + * usually contains. But since our current fastboot + * code puts a mode derived from the post-pfit timings + * into crtc->mode this works out correctly. We don't + * use hwmode anywhere right now, so use it for this + * since the fb helper layer wants a pointer to + * something we own. + */ + DRM_DEBUG_KMS("looking for current mode on connector %s\n", + connector->name); + intel_mode_from_pipe_config(&encoder->crtc->hwmode, + &to_intel_crtc(encoder->crtc)->config); + modes[i] = &encoder->crtc->hwmode; + } + crtcs[i] = new_crtc; + + DRM_DEBUG_KMS("connector %s on pipe %d [CRTC:%d]: %dx%d%s\n", + connector->name, + pipe_name(to_intel_crtc(encoder->crtc)->pipe), + encoder->crtc->base.id, + modes[i]->hdisplay, modes[i]->vdisplay, + modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" :""); + + fallback = false; + } + + /* + * If the BIOS didn't enable everything it could, fall back to have the + * same user experiencing of lighting up as much as possible like the + * fbdev helper library. + */ + if (num_connectors_enabled != num_connectors_detected && + num_connectors_enabled < INTEL_INFO(dev)->num_pipes) { + DRM_DEBUG_KMS("fallback: Not all outputs enabled\n"); + DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled, + num_connectors_detected); + fallback = true; + } + +out: + if (fallback) { + DRM_DEBUG_KMS("Not using firmware configuration\n"); + memcpy(enabled, save_enabled, dev->mode_config.num_connector); + kfree(save_enabled); + return false; + } + + kfree(save_enabled); + return true; +} + static struct drm_fb_helper_funcs intel_fb_helper_funcs = { + .initial_config = intel_fb_initial_config, .gamma_set = intel_crtc_fb_gamma_set, .gamma_get = intel_crtc_fb_gamma_get, .fb_probe = intelfb_create, @@ -258,8 +475,139 @@ static void intel_fbdev_destroy(struct drm_device *dev, drm_fb_helper_fini(&ifbdev->helper); - drm_framebuffer_unregister_private(&ifbdev->ifb.base); - intel_framebuffer_fini(&ifbdev->ifb); + drm_framebuffer_unregister_private(&ifbdev->fb->base); + drm_framebuffer_remove(&ifbdev->fb->base); +} + +/* + * Build an intel_fbdev struct using a BIOS allocated framebuffer, if possible. + * The core display code will have read out the current plane configuration, + * so we use that to figure out if there's an object for us to use as the + * fb, and if so, we re-use it for the fbdev configuration. + * + * Note we only support a single fb shared across pipes for boot (mostly for + * fbcon), so we just find the biggest and use that. + */ +static bool intel_fbdev_init_bios(struct drm_device *dev, + struct intel_fbdev *ifbdev) +{ + struct intel_framebuffer *fb = NULL; + struct drm_crtc *crtc; + struct intel_crtc *intel_crtc; + struct intel_plane_config *plane_config = NULL; + unsigned int max_size = 0; + + if (!i915.fastboot) + return false; + + /* Find the largest fb */ + for_each_crtc(dev, crtc) { + intel_crtc = to_intel_crtc(crtc); + + if (!intel_crtc->active || !crtc->primary->fb) { + DRM_DEBUG_KMS("pipe %c not active or no fb, skipping\n", + pipe_name(intel_crtc->pipe)); + continue; + } + + if (intel_crtc->plane_config.size > max_size) { + DRM_DEBUG_KMS("found possible fb from plane %c\n", + pipe_name(intel_crtc->pipe)); + plane_config = &intel_crtc->plane_config; + fb = to_intel_framebuffer(crtc->primary->fb); + max_size = plane_config->size; + } + } + + if (!fb) { + DRM_DEBUG_KMS("no active fbs found, not using BIOS config\n"); + goto out; + } + + /* Now make sure all the pipes will fit into it */ + for_each_crtc(dev, crtc) { + unsigned int cur_size; + + intel_crtc = to_intel_crtc(crtc); + + if (!intel_crtc->active) { + DRM_DEBUG_KMS("pipe %c not active, skipping\n", + pipe_name(intel_crtc->pipe)); + continue; + } + + DRM_DEBUG_KMS("checking plane %c for BIOS fb\n", + pipe_name(intel_crtc->pipe)); + + /* + * See if the plane fb we found above will fit on this + * pipe. Note we need to use the selected fb's pitch and bpp + * rather than the current pipe's, since they differ. + */ + cur_size = intel_crtc->config.adjusted_mode.crtc_hdisplay; + cur_size = cur_size * fb->base.bits_per_pixel / 8; + if (fb->base.pitches[0] < cur_size) { + DRM_DEBUG_KMS("fb not wide enough for plane %c (%d vs %d)\n", + pipe_name(intel_crtc->pipe), + cur_size, fb->base.pitches[0]); + plane_config = NULL; + fb = NULL; + break; + } + + cur_size = intel_crtc->config.adjusted_mode.crtc_vdisplay; + cur_size = ALIGN(cur_size, plane_config->tiled ? (IS_GEN2(dev) ? 16 : 8) : 1); + cur_size *= fb->base.pitches[0]; + DRM_DEBUG_KMS("pipe %c area: %dx%d, bpp: %d, size: %d\n", + pipe_name(intel_crtc->pipe), + intel_crtc->config.adjusted_mode.crtc_hdisplay, + intel_crtc->config.adjusted_mode.crtc_vdisplay, + fb->base.bits_per_pixel, + cur_size); + + if (cur_size > max_size) { + DRM_DEBUG_KMS("fb not big enough for plane %c (%d vs %d)\n", + pipe_name(intel_crtc->pipe), + cur_size, max_size); + plane_config = NULL; + fb = NULL; + break; + } + + DRM_DEBUG_KMS("fb big enough for plane %c (%d >= %d)\n", + pipe_name(intel_crtc->pipe), + max_size, cur_size); + } + + if (!fb) { + DRM_DEBUG_KMS("BIOS fb not suitable for all pipes, not using\n"); + goto out; + } + + ifbdev->preferred_bpp = fb->base.bits_per_pixel; + ifbdev->fb = fb; + + drm_framebuffer_reference(&ifbdev->fb->base); + + /* Final pass to check if any active pipes don't have fbs */ + for_each_crtc(dev, crtc) { + intel_crtc = to_intel_crtc(crtc); + + if (!intel_crtc->active) + continue; + + WARN(!crtc->primary->fb, + "re-used BIOS config but lost an fb on crtc %d\n", + crtc->base.id); + } + + + DRM_DEBUG_KMS("using BIOS fb for initial console\n"); + return true; + +out: + + return false; } int intel_fbdev_init(struct drm_device *dev) @@ -268,21 +616,25 @@ int intel_fbdev_init(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int ret; - ifbdev = kzalloc(sizeof(*ifbdev), GFP_KERNEL); - if (!ifbdev) + if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0)) + return -ENODEV; + + ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL); + if (ifbdev == NULL) return -ENOMEM; - dev_priv->fbdev = ifbdev; ifbdev->helper.funcs = &intel_fb_helper_funcs; + if (!intel_fbdev_init_bios(dev, ifbdev)) + ifbdev->preferred_bpp = 32; ret = drm_fb_helper_init(dev, &ifbdev->helper, - INTEL_INFO(dev)->num_pipes, - 4); + INTEL_INFO(dev)->num_pipes, 4); if (ret) { kfree(ifbdev); return ret; } + dev_priv->fbdev = ifbdev; drm_fb_helper_single_add_all_connectors(&ifbdev->helper); return 0; @@ -291,9 +643,10 @@ int intel_fbdev_init(struct drm_device *dev) void intel_fbdev_initial_config(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_fbdev *ifbdev = dev_priv->fbdev; /* Due to peculiar init order wrt to hpd handling this is separate. */ - drm_fb_helper_initial_config(&dev_priv->fbdev->helper, 32); + drm_fb_helper_initial_config(&ifbdev->helper, ifbdev->preferred_bpp); } void intel_fbdev_fini(struct drm_device *dev) @@ -322,7 +675,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state) * been restored from swap. If the object is stolen however, it will be * full of whatever garbage was left in there. */ - if (state == FBINFO_STATE_RUNNING && ifbdev->ifb.obj->stolen) + if (state == FBINFO_STATE_RUNNING && ifbdev->fb->obj->stolen) memset_io(info->screen_base, 0, info->screen_size); fb_set_suspend(info, state); @@ -331,7 +684,8 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state) void intel_fbdev_output_poll_changed(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper); + if (dev_priv->fbdev) + drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper); } void intel_fbdev_restore_mode(struct drm_device *dev) @@ -339,14 +693,10 @@ void intel_fbdev_restore_mode(struct drm_device *dev) int ret; struct drm_i915_private *dev_priv = dev->dev_private; - if (INTEL_INFO(dev)->num_pipes == 0) + if (!dev_priv->fbdev) return; - drm_modeset_lock_all(dev); - - ret = drm_fb_helper_restore_fbdev_mode(&dev_priv->fbdev->helper); + ret = drm_fb_helper_restore_fbdev_mode_unlocked(&dev_priv->fbdev->helper); if (ret) DRM_DEBUG("failed to restore crtc mode\n"); - - drm_modeset_unlock_all(dev); } diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index 6db0d9d17f4..eee2bbec295 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -113,7 +113,8 @@ static u32 hsw_infoframe_enable(enum hdmi_infoframe_type type) } static u32 hsw_infoframe_data_reg(enum hdmi_infoframe_type type, - enum transcoder cpu_transcoder) + enum transcoder cpu_transcoder, + struct drm_i915_private *dev_priv) { switch (type) { case HDMI_INFOFRAME_TYPE_AVI: @@ -296,7 +297,8 @@ static void hsw_write_infoframe(struct drm_encoder *encoder, u32 val = I915_READ(ctl_reg); data_reg = hsw_infoframe_data_reg(type, - intel_crtc->config.cpu_transcoder); + intel_crtc->config.cpu_transcoder, + dev_priv); if (data_reg == 0) return; @@ -416,6 +418,7 @@ intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder, } static void g4x_set_infoframes(struct drm_encoder *encoder, + bool enable, struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; @@ -423,7 +426,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; u32 reg = VIDEO_DIP_CTL; u32 val = I915_READ(reg); - u32 port; + u32 port = VIDEO_DIP_PORT(intel_dig_port->port); assert_hdmi_port_disabled(intel_hdmi); @@ -438,7 +441,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, * either. */ val |= VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC; - if (!intel_hdmi->has_hdmi_sink) { + if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; val &= ~VIDEO_DIP_ENABLE; @@ -447,18 +450,6 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, return; } - switch (intel_dig_port->port) { - case PORT_B: - port = VIDEO_DIP_PORT_B; - break; - case PORT_C: - port = VIDEO_DIP_PORT_C; - break; - default: - BUG(); - return; - } - if (port != (val & VIDEO_DIP_PORT_MASK)) { if (val & VIDEO_DIP_ENABLE) { val &= ~VIDEO_DIP_ENABLE; @@ -481,6 +472,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, } static void ibx_set_infoframes(struct drm_encoder *encoder, + bool enable, struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; @@ -489,14 +481,14 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; u32 reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); - u32 port; + u32 port = VIDEO_DIP_PORT(intel_dig_port->port); assert_hdmi_port_disabled(intel_hdmi); /* See the big comment in g4x_set_infoframes() */ val |= VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC; - if (!intel_hdmi->has_hdmi_sink) { + if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; val &= ~VIDEO_DIP_ENABLE; @@ -505,21 +497,6 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, return; } - switch (intel_dig_port->port) { - case PORT_B: - port = VIDEO_DIP_PORT_B; - break; - case PORT_C: - port = VIDEO_DIP_PORT_C; - break; - case PORT_D: - port = VIDEO_DIP_PORT_D; - break; - default: - BUG(); - return; - } - if (port != (val & VIDEO_DIP_PORT_MASK)) { if (val & VIDEO_DIP_ENABLE) { val &= ~VIDEO_DIP_ENABLE; @@ -543,6 +520,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, } static void cpt_set_infoframes(struct drm_encoder *encoder, + bool enable, struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; @@ -556,7 +534,7 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, /* See the big comment in g4x_set_infoframes() */ val |= VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC; - if (!intel_hdmi->has_hdmi_sink) { + if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI); @@ -579,20 +557,23 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, } static void vlv_set_infoframes(struct drm_encoder *encoder, + bool enable, struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); u32 reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); + u32 port = VIDEO_DIP_PORT(intel_dig_port->port); assert_hdmi_port_disabled(intel_hdmi); /* See the big comment in g4x_set_infoframes() */ val |= VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC; - if (!intel_hdmi->has_hdmi_sink) { + if (!enable) { if (!(val & VIDEO_DIP_ENABLE)) return; val &= ~VIDEO_DIP_ENABLE; @@ -601,9 +582,19 @@ static void vlv_set_infoframes(struct drm_encoder *encoder, return; } + if (port != (val & VIDEO_DIP_PORT_MASK)) { + if (val & VIDEO_DIP_ENABLE) { + val &= ~VIDEO_DIP_ENABLE; + I915_WRITE(reg, val); + POSTING_READ(reg); + } + val &= ~VIDEO_DIP_PORT_MASK; + val |= port; + } + val |= VIDEO_DIP_ENABLE; - val &= ~(VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT | - VIDEO_DIP_ENABLE_GCP); + val &= ~(VIDEO_DIP_ENABLE_AVI | VIDEO_DIP_ENABLE_VENDOR | + VIDEO_DIP_ENABLE_GAMUT | VIDEO_DIP_ENABLE_GCP); I915_WRITE(reg, val); POSTING_READ(reg); @@ -614,6 +605,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder, } static void hsw_set_infoframes(struct drm_encoder *encoder, + bool enable, struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = encoder->dev->dev_private; @@ -624,7 +616,7 @@ static void hsw_set_infoframes(struct drm_encoder *encoder, assert_hdmi_port_disabled(intel_hdmi); - if (!intel_hdmi->has_hdmi_sink) { + if (!enable) { I915_WRITE(reg, 0); POSTING_READ(reg); return; @@ -641,7 +633,7 @@ static void hsw_set_infoframes(struct drm_encoder *encoder, intel_hdmi_set_hdmi_infoframe(encoder, adjusted_mode); } -static void intel_hdmi_mode_set(struct intel_encoder *encoder) +static void intel_hdmi_prepare(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -663,27 +655,26 @@ static void intel_hdmi_mode_set(struct intel_encoder *encoder) else hdmi_val |= SDVO_COLOR_FORMAT_8bpc; - /* Required on CPT */ - if (intel_hdmi->has_hdmi_sink && HAS_PCH_CPT(dev)) + if (crtc->config.has_hdmi_sink) hdmi_val |= HDMI_MODE_SELECT_HDMI; - if (intel_hdmi->has_audio) { + if (crtc->config.has_audio) { + WARN_ON(!crtc->config.has_hdmi_sink); DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n", pipe_name(crtc->pipe)); hdmi_val |= SDVO_AUDIO_ENABLE; - hdmi_val |= HDMI_MODE_SELECT_HDMI; intel_write_eld(&encoder->base, adjusted_mode); } if (HAS_PCH_CPT(dev)) hdmi_val |= SDVO_PIPE_SEL_CPT(crtc->pipe); + else if (IS_CHERRYVIEW(dev)) + hdmi_val |= SDVO_PIPE_SEL_CHV(crtc->pipe); else hdmi_val |= SDVO_PIPE_SEL(crtc->pipe); I915_WRITE(intel_hdmi->hdmi_reg, hdmi_val); POSTING_READ(intel_hdmi->hdmi_reg); - - intel_hdmi->set_infoframes(&encoder->base, adjusted_mode); } static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder, @@ -692,8 +683,13 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder, struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); + enum intel_display_power_domain power_domain; u32 tmp; + power_domain = intel_display_port_power_domain(encoder); + if (!intel_display_power_enabled(dev_priv, power_domain)) + return false; + tmp = I915_READ(intel_hdmi->hdmi_reg); if (!(tmp & SDVO_ENABLE)) @@ -701,6 +697,8 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder, if (HAS_PCH_CPT(dev)) *pipe = PORT_TO_PIPE_CPT(tmp); + else if (IS_CHERRYVIEW(dev)) + *pipe = SDVO_PORT_TO_PIPE_CHV(tmp); else *pipe = PORT_TO_PIPE(tmp); @@ -727,6 +725,12 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, else flags |= DRM_MODE_FLAG_NVSYNC; + if (tmp & HDMI_MODE_SELECT_HDMI) + pipe_config->has_hdmi_sink = true; + + if (tmp & HDMI_MODE_SELECT_HDMI) + pipe_config->has_audio = true; + pipe_config->adjusted_mode.flags |= flags; if ((tmp & SDVO_COLOR_FORMAT_MASK) == HDMI_COLOR_FORMAT_12bpc) @@ -749,7 +753,7 @@ static void intel_enable_hdmi(struct intel_encoder *encoder) u32 temp; u32 enable_bits = SDVO_ENABLE; - if (intel_hdmi->has_audio) + if (intel_crtc->config.has_audio) enable_bits |= SDVO_AUDIO_ENABLE; temp = I915_READ(intel_hdmi->hdmi_reg); @@ -841,11 +845,11 @@ static void intel_disable_hdmi(struct intel_encoder *encoder) } } -static int hdmi_portclock_limit(struct intel_hdmi *hdmi) +static int hdmi_portclock_limit(struct intel_hdmi *hdmi, bool respect_dvi_limit) { struct drm_device *dev = intel_hdmi_to_dev(hdmi); - if (IS_G4X(dev)) + if ((respect_dvi_limit && !hdmi->has_hdmi_sink) || IS_G4X(dev)) return 165000; else if (IS_HASWELL(dev) || INTEL_INFO(dev)->gen >= 8) return 300000; @@ -857,7 +861,8 @@ static enum drm_mode_status intel_hdmi_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { - if (mode->clock > hdmi_portclock_limit(intel_attached_hdmi(connector))) + if (mode->clock > hdmi_portclock_limit(intel_attached_hdmi(connector), + true)) return MODE_CLOCK_HIGH; if (mode->clock < 20000) return MODE_CLOCK_LOW; @@ -868,6 +873,30 @@ intel_hdmi_mode_valid(struct drm_connector *connector, return MODE_OK; } +static bool hdmi_12bpc_possible(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct intel_encoder *encoder; + int count = 0, count_hdmi = 0; + + if (!HAS_PCH_SPLIT(dev)) + return false; + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) { + if (encoder->new_crtc != crtc) + continue; + + count_hdmi += encoder->type == INTEL_OUTPUT_HDMI; + count++; + } + + /* + * HDMI 12bpc affects the clocks, so it's only possible + * when not cloning with other encoder types. + */ + return count_hdmi > 0 && count_hdmi == count; +} + bool intel_hdmi_compute_config(struct intel_encoder *encoder, struct intel_crtc_config *pipe_config) { @@ -875,12 +904,14 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, struct drm_device *dev = encoder->base.dev; struct drm_display_mode *adjusted_mode = &pipe_config->adjusted_mode; int clock_12bpc = pipe_config->adjusted_mode.crtc_clock * 3 / 2; - int portclock_limit = hdmi_portclock_limit(intel_hdmi); + int portclock_limit = hdmi_portclock_limit(intel_hdmi, false); int desired_bpp; + pipe_config->has_hdmi_sink = intel_hdmi->has_hdmi_sink; + if (intel_hdmi->color_range_auto) { /* See CEA-861-E - 5.1 Default Encoding Parameters */ - if (intel_hdmi->has_hdmi_sink && + if (pipe_config->has_hdmi_sink && drm_match_cea_mode(adjusted_mode) > 1) intel_hdmi->color_range = HDMI_COLOR_RANGE_16_235; else @@ -893,14 +924,18 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder, if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev)) pipe_config->has_pch_encoder = true; + if (pipe_config->has_hdmi_sink && intel_hdmi->has_audio) + pipe_config->has_audio = true; + /* * HDMI is either 12 or 8, so if the display lets 10bpc sneak * through, clamp it down. Note that g4x/vlv don't support 12bpc hdmi * outputs. We also need to check that the higher clock still fits * within limits. */ - if (pipe_config->pipe_bpp > 8*3 && clock_12bpc <= portclock_limit - && HAS_PCH_SPLIT(dev)) { + if (pipe_config->pipe_bpp > 8*3 && pipe_config->has_hdmi_sink && + clock_12bpc <= portclock_limit && + hdmi_12bpc_possible(encoder->new_crtc)) { DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n"); desired_bpp = 12*3; @@ -934,10 +969,14 @@ intel_hdmi_detect(struct drm_connector *connector, bool force) struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_i915_private *dev_priv = dev->dev_private; struct edid *edid; + enum intel_display_power_domain power_domain; enum drm_connector_status status = connector_status_disconnected; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); + + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); intel_hdmi->has_hdmi_sink = false; intel_hdmi->has_audio = false; @@ -966,31 +1005,48 @@ intel_hdmi_detect(struct drm_connector *connector, bool force) intel_encoder->type = INTEL_OUTPUT_HDMI; } + intel_display_power_put(dev_priv, power_domain); + return status; } static int intel_hdmi_get_modes(struct drm_connector *connector) { - struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + struct intel_encoder *intel_encoder = intel_attached_encoder(connector); + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); struct drm_i915_private *dev_priv = connector->dev->dev_private; + enum intel_display_power_domain power_domain; + int ret; /* We should parse the EDID data and find out if it's an HDMI sink so * we can send audio to it. */ - return intel_ddc_get_modes(connector, + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + + ret = intel_ddc_get_modes(connector, intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus)); + + intel_display_power_put(dev_priv, power_domain); + + return ret; } static bool intel_hdmi_detect_audio(struct drm_connector *connector) { - struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + struct intel_encoder *intel_encoder = intel_attached_encoder(connector); + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base); struct drm_i915_private *dev_priv = connector->dev->dev_private; + enum intel_display_power_domain power_domain; struct edid *edid; bool has_audio = false; + power_domain = intel_display_port_power_domain(intel_encoder); + intel_display_power_get(dev_priv, power_domain); + edid = drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus)); @@ -1000,6 +1056,8 @@ intel_hdmi_detect_audio(struct drm_connector *connector) kfree(edid); } + intel_display_power_put(dev_priv, power_domain); + return has_audio; } @@ -1075,20 +1133,34 @@ done: return 0; } +static void intel_hdmi_pre_enable(struct intel_encoder *encoder) +{ + struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); + struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + struct drm_display_mode *adjusted_mode = + &intel_crtc->config.adjusted_mode; + + intel_hdmi_prepare(encoder); + + intel_hdmi->set_infoframes(&encoder->base, + intel_crtc->config.has_hdmi_sink, + adjusted_mode); +} + static void vlv_hdmi_pre_enable(struct intel_encoder *encoder) { struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct intel_hdmi *intel_hdmi = &dport->hdmi; struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + struct drm_display_mode *adjusted_mode = + &intel_crtc->config.adjusted_mode; enum dpio_channel port = vlv_dport_to_channel(dport); int pipe = intel_crtc->pipe; u32 val; - if (!IS_VALLEYVIEW(dev)) - return; - /* Enable clock channels for this port */ mutex_lock(&dev_priv->dpio_lock); val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port)); @@ -1115,6 +1187,10 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder) vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888); mutex_unlock(&dev_priv->dpio_lock); + intel_hdmi->set_infoframes(&encoder->base, + intel_crtc->config.has_hdmi_sink, + adjusted_mode); + intel_enable_hdmi(encoder); vlv_wait_port_ready(dev_priv, dport); @@ -1130,8 +1206,7 @@ static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder) enum dpio_channel port = vlv_dport_to_channel(dport); int pipe = intel_crtc->pipe; - if (!IS_VALLEYVIEW(dev)) - return; + intel_hdmi_prepare(encoder); /* Program Tx lane resets to default */ mutex_lock(&dev_priv->dpio_lock); @@ -1170,6 +1245,152 @@ static void vlv_hdmi_post_disable(struct intel_encoder *encoder) mutex_unlock(&dev_priv->dpio_lock); } +static void chv_hdmi_post_disable(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + enum pipe pipe = intel_crtc->pipe; + u32 val; + + mutex_lock(&dev_priv->dpio_lock); + + /* Propagate soft reset to data lane reset */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + + mutex_unlock(&dev_priv->dpio_lock); +} + +static void chv_hdmi_pre_enable(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + int data, i; + u32 val; + + mutex_lock(&dev_priv->dpio_lock); + + /* Deassert soft data lane reset*/ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + val |= (DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + + /* Program Tx latency optimal setting */ + for (i = 0; i < 4; i++) { + /* Set the latency optimal bit */ + data = (i == 1) ? 0x0 : 0x6; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW11(ch, i), + data << DPIO_FRC_LATENCY_SHFIT); + + /* Set the upar bit */ + data = (i == 1) ? 0x0 : 0x1; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), + data << DPIO_UPAR_SHIFT); + } + + /* Data lane stagger programming */ + /* FIXME: Fix up value only after power analysis */ + + /* Clear calc init */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + + /* FIXME: Program the support xxx V-dB */ + /* Use 800mV-0dB */ + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); + val &= ~DPIO_SWING_DEEMPH9P5_MASK; + val |= 128 << DPIO_SWING_DEEMPH9P5_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); + } + + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); + val &= ~DPIO_SWING_MARGIN_MASK; + val |= 102 << DPIO_SWING_MARGIN_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); + } + + /* Disable unique transition scale */ + for (i = 0; i < 4; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); + val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); + } + + /* Additional steps for 1200mV-0dB */ +#if 0 + val = vlv_dpio_read(dev_priv, pipe, VLV_TX_DW3(ch)); + if (ch) + val |= DPIO_TX_UNIQ_TRANS_SCALE_CH1; + else + val |= DPIO_TX_UNIQ_TRANS_SCALE_CH0; + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(ch), val); + + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(ch), + vlv_dpio_read(dev_priv, pipe, VLV_TX_DW2(ch)) | + (0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT)); +#endif + /* Start swing calculation */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + + /* LRC Bypass */ + val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30); + val |= DPIO_LRC_BYPASS; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, val); + + mutex_unlock(&dev_priv->dpio_lock); + + intel_enable_hdmi(encoder); + + vlv_wait_port_ready(dev_priv, dport); +} + static void intel_hdmi_destroy(struct drm_connector *connector) { drm_connector_cleanup(connector); @@ -1230,7 +1451,10 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_encoder->hpd_pin = HPD_PORT_C; break; case PORT_D: - intel_hdmi->ddc_bus = GMBUS_PORT_DPD; + if (IS_CHERRYVIEW(dev)) + intel_hdmi->ddc_bus = GMBUS_PORT_DPD_CHV; + else + intel_hdmi->ddc_bus = GMBUS_PORT_DPD; intel_encoder->hpd_pin = HPD_PORT_D; break; case PORT_A: @@ -1261,6 +1485,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; else intel_connector->get_hw_state = intel_connector_get_hw_state; + intel_connector->unregister = intel_connector_unregister; intel_hdmi_add_properties(intel_hdmi, connector); @@ -1299,22 +1524,40 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port) DRM_MODE_ENCODER_TMDS); intel_encoder->compute_config = intel_hdmi_compute_config; - intel_encoder->mode_set = intel_hdmi_mode_set; intel_encoder->disable = intel_disable_hdmi; intel_encoder->get_hw_state = intel_hdmi_get_hw_state; intel_encoder->get_config = intel_hdmi_get_config; - if (IS_VALLEYVIEW(dev)) { + if (IS_CHERRYVIEW(dev)) { + intel_encoder->pre_enable = chv_hdmi_pre_enable; + intel_encoder->enable = vlv_enable_hdmi; + intel_encoder->post_disable = chv_hdmi_post_disable; + } else if (IS_VALLEYVIEW(dev)) { intel_encoder->pre_pll_enable = vlv_hdmi_pre_pll_enable; intel_encoder->pre_enable = vlv_hdmi_pre_enable; intel_encoder->enable = vlv_enable_hdmi; intel_encoder->post_disable = vlv_hdmi_post_disable; } else { + intel_encoder->pre_enable = intel_hdmi_pre_enable; intel_encoder->enable = intel_enable_hdmi; } intel_encoder->type = INTEL_OUTPUT_HDMI; - intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); - intel_encoder->cloneable = false; + if (IS_CHERRYVIEW(dev)) { + if (port == PORT_D) + intel_encoder->crtc_mask = 1 << 2; + else + intel_encoder->crtc_mask = (1 << 0) | (1 << 1); + } else { + intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); + } + intel_encoder->cloneable = 1 << INTEL_OUTPUT_ANALOG; + /* + * BSpec is unclear about HDMI+HDMI cloning on g4x, but it seems + * to work on real hardware. And since g4x can send infoframes to + * only one port anyway, nothing is lost by allowing it. + */ + if (IS_G4X(dev)) + intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI; intel_dig_port->port = port; intel_dig_port->hdmi.hdmi_reg = hdmi_reg; diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c index b1dc33f4789..d33b61d0dd3 100644 --- a/drivers/gpu/drm/i915/intel_i2c.c +++ b/drivers/gpu/drm/i915/intel_i2c.c @@ -258,13 +258,6 @@ intel_gpio_setup(struct intel_gmbus *bus, u32 pin) algo->data = bus; } -/* - * gmbus on gen4 seems to be able to generate legacy interrupts even when in MSI - * mode. This results in spurious interrupt warnings if the legacy irq no. is - * shared with another device. The kernel then disables that interrupt source - * and so prevents the other device from working properly. - */ -#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->gen >= 5) static int gmbus_wait_hw_status(struct drm_i915_private *dev_priv, u32 gmbus2_status, diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 8bcb93a2a9f..5e5a72fca5f 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -126,10 +126,6 @@ static void intel_lvds_get_config(struct intel_encoder *encoder, pipe_config->adjusted_mode.crtc_clock = dotclock; } -/* The LVDS pin pair needs to be on before the DPLLs are enabled. - * This is an exception to the general rule that mode_set doesn't turn - * things on. - */ static void intel_pre_enable_lvds(struct intel_encoder *encoder) { struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); @@ -331,15 +327,6 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder, return true; } -static void intel_lvds_mode_set(struct intel_encoder *encoder) -{ - /* - * We don't do anything here, the LVDS port is fully set up in the pre - * enable hook - the ordering constraints for enabling the lvds port vs. - * enabling the display pll are too strict. - */ -} - /** * Detect the LVDS connection. * @@ -354,7 +341,7 @@ intel_lvds_detect(struct drm_connector *connector, bool force) enum drm_connector_status status; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); status = intel_panel_detect(dev); if (status != connector_status_unknown) @@ -848,8 +835,8 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder) struct drm_i915_private *dev_priv = dev->dev_private; /* use the module option value if specified */ - if (i915_lvds_channel_mode > 0) - return i915_lvds_channel_mode == 2; + if (i915.lvds_channel_mode > 0) + return i915.lvds_channel_mode == 2; if (dmi_check_system(intel_dual_link_lvds)) return true; @@ -899,6 +886,7 @@ void intel_lvds_init(struct drm_device *dev) struct drm_encoder *encoder; struct drm_display_mode *scan; /* *modes, *bios_mode; */ struct drm_display_mode *fixed_mode = NULL; + struct drm_display_mode *downclock_mode = NULL; struct edid *edid; struct drm_crtc *crtc; u32 lvds; @@ -952,16 +940,16 @@ void intel_lvds_init(struct drm_device *dev) intel_encoder->enable = intel_enable_lvds; intel_encoder->pre_enable = intel_pre_enable_lvds; intel_encoder->compute_config = intel_lvds_compute_config; - intel_encoder->mode_set = intel_lvds_mode_set; intel_encoder->disable = intel_disable_lvds; intel_encoder->get_hw_state = intel_lvds_get_hw_state; intel_encoder->get_config = intel_lvds_get_config; intel_connector->get_hw_state = intel_connector_get_hw_state; + intel_connector->unregister = intel_connector_unregister; intel_connector_attach_encoder(intel_connector, intel_encoder); intel_encoder->type = INTEL_OUTPUT_LVDS; - intel_encoder->cloneable = false; + intel_encoder->cloneable = 0; if (HAS_PCH_SPLIT(dev)) intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); else if (IS_GEN4(dev)) @@ -1000,6 +988,7 @@ void intel_lvds_init(struct drm_device *dev) * Attempt to get the fixed panel mode from DDC. Assume that the * preferred mode is the right one. */ + mutex_lock(&dev->mode_config.mutex); edid = drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv, pin)); if (edid) { if (drm_add_edid_modes(connector, edid)) { @@ -1032,15 +1021,14 @@ void intel_lvds_init(struct drm_device *dev) fixed_mode = drm_mode_duplicate(dev, scan); if (fixed_mode) { - intel_connector->panel.downclock_mode = + downclock_mode = intel_find_panel_downclock(dev, fixed_mode, connector); - if (intel_connector->panel.downclock_mode != - NULL && i915_lvds_downclock) { + if (downclock_mode != NULL && + i915.lvds_downclock) { /* We found the downclock for LVDS. */ dev_priv->lvds_downclock_avail = true; dev_priv->lvds_downclock = - intel_connector->panel. downclock_mode->clock; DRM_DEBUG_KMS("LVDS downclock is found" " in EDID. Normal clock %dKhz, " @@ -1094,6 +1082,8 @@ void intel_lvds_init(struct drm_device *dev) goto failed; out: + mutex_unlock(&dev->mode_config.mutex); + lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder); DRM_DEBUG_KMS("detected %s-link lvds configuration\n", lvds_encoder->is_dual_link ? "dual" : "single"); @@ -1116,17 +1106,17 @@ out: } drm_sysfs_connector_add(connector); - intel_panel_init(&intel_connector->panel, fixed_mode); + intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); intel_panel_setup_backlight(connector); return; failed: + mutex_unlock(&dev->mode_config.mutex); + DRM_DEBUG_KMS("No LVDS modes found, disabling.\n"); drm_connector_cleanup(connector); drm_encoder_cleanup(encoder); - if (fixed_mode) - drm_mode_destroy(dev, fixed_mode); kfree(lvds_encoder); kfree(lvds_connector); return; diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 4e960ec7419..4f6b53998d7 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -226,6 +226,8 @@ struct opregion_asle { #define ACPI_DIGITAL_OUTPUT (3<<8) #define ACPI_LVDS_OUTPUT (4<<8) +#define MAX_DSLP 1500 + #ifdef CONFIG_ACPI static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) { @@ -260,10 +262,11 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) /* The spec says 2ms should be the default, but it's too small * for some machines. */ dslp = 50; - } else if (dslp > 500) { + } else if (dslp > MAX_DSLP) { /* Hey bios, trust must be earned. */ - WARN_ONCE(1, "excessive driver sleep timeout (DSPL) %u\n", dslp); - dslp = 500; + DRM_INFO_ONCE("ACPI BIOS requests an excessive sleep of %u ms, " + "using %u ms instead\n", dslp, MAX_DSLP); + dslp = MAX_DSLP; } /* The spec tells us to do this, but we are the only user... */ @@ -400,6 +403,15 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp); + /* + * If the acpi_video interface is not supposed to be used, don't + * bother processing backlight level change requests from firmware. + */ + if (!acpi_video_verify_backlight_support()) { + DRM_DEBUG_KMS("opregion backlight request ignored\n"); + return 0; + } + if (!(bclp & ASLE_BCLP_VALID)) return ASLC_BACKLIGHT_FAILED; @@ -407,7 +419,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) if (bclp > 255) return ASLC_BACKLIGHT_FAILED; - mutex_lock(&dev->mode_config.mutex); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); /* * Update backlight on all connectors that support backlight (usually @@ -418,7 +430,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) intel_panel_set_backlight(intel_connector, bclp, 255); iowrite32(DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID, &asle->cblv); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_unlock(&dev->mode_config.connection_mutex); return 0; diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index a759ecdb7a6..daa118978ee 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -189,11 +189,11 @@ struct intel_overlay { static struct overlay_registers __iomem * intel_overlay_map_regs(struct intel_overlay *overlay) { - drm_i915_private_t *dev_priv = overlay->dev->dev_private; + struct drm_i915_private *dev_priv = overlay->dev->dev_private; struct overlay_registers __iomem *regs; if (OVERLAY_NEEDS_PHYSICAL(overlay->dev)) - regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_obj->handle->vaddr; + regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_handle->vaddr; else regs = io_mapping_map_wc(dev_priv->gtt.mappable, i915_gem_obj_ggtt_offset(overlay->reg_bo)); @@ -212,8 +212,8 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay, void (*tail)(struct intel_overlay *)) { struct drm_device *dev = overlay->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; int ret; BUG_ON(overlay->last_flip_req); @@ -236,7 +236,7 @@ static int intel_overlay_on(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; int ret; BUG_ON(overlay->active); @@ -262,8 +262,8 @@ static int intel_overlay_continue(struct intel_overlay *overlay, bool load_polyphase_filter) { struct drm_device *dev = overlay->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; u32 flip_addr = overlay->flip_addr; u32 tmp; int ret; @@ -293,7 +293,7 @@ static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay) { struct drm_i915_gem_object *obj = overlay->old_vid_bo; - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); drm_gem_object_unreference(&obj->base); overlay->old_vid_bo = NULL; @@ -306,7 +306,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay) /* never have the overlay hw on without showing a frame */ BUG_ON(!overlay->vid_bo); - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); drm_gem_object_unreference(&obj->base); overlay->vid_bo = NULL; @@ -320,7 +320,7 @@ static int intel_overlay_off(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; u32 flip_addr = overlay->flip_addr; int ret; @@ -362,8 +362,8 @@ static int intel_overlay_off(struct intel_overlay *overlay) static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; int ret; if (overlay->last_flip_req == 0) @@ -388,8 +388,8 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) static int intel_overlay_release_old_vid(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; int ret; /* Only wait if there is actually an old frame to release to @@ -606,14 +606,14 @@ static void update_colorkey(struct intel_overlay *overlay, { u32 key = overlay->color_key; - switch (overlay->crtc->base.fb->bits_per_pixel) { + switch (overlay->crtc->base.primary->fb->bits_per_pixel) { case 8: iowrite32(0, ®s->DCLRKV); iowrite32(CLK_RGB8I_MASK | DST_KEY_ENABLE, ®s->DCLRKM); break; case 16: - if (overlay->crtc->base.fb->depth == 15) { + if (overlay->crtc->base.primary->fb->depth == 15) { iowrite32(RGB15_TO_COLORKEY(key), ®s->DCLRKV); iowrite32(CLK_RGB15_MASK | DST_KEY_ENABLE, ®s->DCLRKM); @@ -688,7 +688,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, u32 swidth, swidthsw, sheight, ostride; BUG_ON(!mutex_is_locked(&dev->struct_mutex)); - BUG_ON(!mutex_is_locked(&dev->mode_config.mutex)); + BUG_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); BUG_ON(!overlay); ret = intel_overlay_release_old_vid(overlay); @@ -782,7 +782,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, return 0; out_unpin: - i915_gem_object_unpin(new_bo); + i915_gem_object_ggtt_unpin(new_bo); return ret; } @@ -793,7 +793,7 @@ int intel_overlay_switch_off(struct intel_overlay *overlay) int ret; BUG_ON(!mutex_is_locked(&dev->struct_mutex)); - BUG_ON(!mutex_is_locked(&dev->mode_config.mutex)); + BUG_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); ret = intel_overlay_recover_from_interrupt(overlay); if (ret != 0) @@ -834,7 +834,7 @@ static int check_overlay_possible_on_crtc(struct intel_overlay *overlay, static void update_pfit_vscale_ratio(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 pfit_control = I915_READ(PFIT_CONTROL); u32 ratio; @@ -1026,7 +1026,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_intel_overlay_put_image *put_image_rec = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_overlay *overlay; struct drm_mode_object *drmmode_obj; struct intel_crtc *crtc; @@ -1076,7 +1076,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, mutex_lock(&dev->struct_mutex); if (new_bo->tiling_mode) { - DRM_ERROR("buffer used for overlay image can not be tiled\n"); + DRM_DEBUG_KMS("buffer used for overlay image can not be tiled\n"); ret = -EINVAL; goto out_unlock; } @@ -1226,7 +1226,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_intel_overlay_attrs *attrs = data; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_overlay *overlay; struct overlay_registers __iomem *regs; int ret; @@ -1311,7 +1311,7 @@ out_unlock: void intel_setup_overlay(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_overlay *overlay; struct drm_i915_gem_object *reg_bo; struct overlay_registers __iomem *regs; @@ -1340,16 +1340,14 @@ void intel_setup_overlay(struct drm_device *dev) overlay->reg_bo = reg_bo; if (OVERLAY_NEEDS_PHYSICAL(dev)) { - ret = i915_gem_attach_phys_object(dev, reg_bo, - I915_GEM_PHYS_OVERLAY_REGS, - PAGE_SIZE); + ret = i915_gem_object_attach_phys(reg_bo, PAGE_SIZE); if (ret) { DRM_ERROR("failed to attach phys overlay regs\n"); goto out_free_bo; } - overlay->flip_addr = reg_bo->phys_obj->handle->busaddr; + overlay->flip_addr = reg_bo->phys_handle->busaddr; } else { - ret = i915_gem_obj_ggtt_pin(reg_bo, PAGE_SIZE, true, false); + ret = i915_gem_obj_ggtt_pin(reg_bo, PAGE_SIZE, PIN_MAPPABLE); if (ret) { DRM_ERROR("failed to pin overlay register bo\n"); goto out_free_bo; @@ -1386,7 +1384,7 @@ void intel_setup_overlay(struct drm_device *dev) out_unpin_bo: if (!OVERLAY_NEEDS_PHYSICAL(dev)) - i915_gem_object_unpin(reg_bo); + i915_gem_object_ggtt_unpin(reg_bo); out_free_bo: drm_gem_object_unreference(®_bo->base); out_free: @@ -1397,7 +1395,7 @@ out_free: void intel_cleanup_overlay(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; if (!dev_priv->overlay) return; @@ -1421,14 +1419,14 @@ struct intel_overlay_error_state { static struct overlay_registers __iomem * intel_overlay_map_regs_atomic(struct intel_overlay *overlay) { - drm_i915_private_t *dev_priv = overlay->dev->dev_private; + struct drm_i915_private *dev_priv = overlay->dev->dev_private; struct overlay_registers __iomem *regs; if (OVERLAY_NEEDS_PHYSICAL(overlay->dev)) /* Cast to make sparse happy, but it's wc memory anyway, so * equivalent to the wc io mapping on X86. */ regs = (struct overlay_registers __iomem *) - overlay->reg_bo->phys_obj->handle->vaddr; + overlay->reg_bo->phys_handle->vaddr; else regs = io_mapping_map_atomic_wc(dev_priv->gtt.mappable, i915_gem_obj_ggtt_offset(overlay->reg_bo)); @@ -1447,7 +1445,7 @@ static void intel_overlay_unmap_regs_atomic(struct intel_overlay *overlay, struct intel_overlay_error_state * intel_overlay_capture_error_state(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; struct intel_overlay *overlay = dev_priv->overlay; struct intel_overlay_error_state *error; struct overlay_registers __iomem *regs; @@ -1462,7 +1460,7 @@ intel_overlay_capture_error_state(struct drm_device *dev) error->dovsta = I915_READ(DOVSTA); error->isr = I915_READ(ISR); if (OVERLAY_NEEDS_PHYSICAL(overlay->dev)) - error->base = (__force long)overlay->reg_bo->phys_obj->handle->vaddr; + error->base = (__force long)overlay->reg_bo->phys_handle->vaddr; else error->base = i915_gem_obj_ggtt_offset(overlay->reg_bo); diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 350de359123..12b02fe1d0a 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -33,8 +33,6 @@ #include <linux/moduleparam.h> #include "intel_drv.h" -#define PCI_LBPC 0xf4 /* legacy/combination backlight modes */ - void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, struct drm_display_mode *adjusted_mode) @@ -44,6 +42,59 @@ intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, drm_mode_set_crtcinfo(adjusted_mode, 0); } +/** + * intel_find_panel_downclock - find the reduced downclock for LVDS in EDID + * @dev: drm device + * @fixed_mode : panel native mode + * @connector: LVDS/eDP connector + * + * Return downclock_avail + * Find the reduced downclock for LVDS/eDP in EDID. + */ +struct drm_display_mode * +intel_find_panel_downclock(struct drm_device *dev, + struct drm_display_mode *fixed_mode, + struct drm_connector *connector) +{ + struct drm_display_mode *scan, *tmp_mode; + int temp_downclock; + + temp_downclock = fixed_mode->clock; + tmp_mode = NULL; + + list_for_each_entry(scan, &connector->probed_modes, head) { + /* + * If one mode has the same resolution with the fixed_panel + * mode while they have the different refresh rate, it means + * that the reduced downclock is found. In such + * case we can set the different FPx0/1 to dynamically select + * between low and high frequency. + */ + if (scan->hdisplay == fixed_mode->hdisplay && + scan->hsync_start == fixed_mode->hsync_start && + scan->hsync_end == fixed_mode->hsync_end && + scan->htotal == fixed_mode->htotal && + scan->vdisplay == fixed_mode->vdisplay && + scan->vsync_start == fixed_mode->vsync_start && + scan->vsync_end == fixed_mode->vsync_end && + scan->vtotal == fixed_mode->vtotal) { + if (scan->clock < temp_downclock) { + /* + * The downclock is already found. But we + * expect to find the lower downclock. + */ + temp_downclock = scan->clock; + tmp_mode = scan; + } + } + } + + if (temp_downclock < fixed_mode->clock) + return drm_mode_duplicate(dev, tmp_mode); + else + return NULL; +} + /* adjusted_mode has been preset to be the panel's fixed mode */ void intel_pch_panel_fitting(struct intel_crtc *intel_crtc, @@ -325,13 +376,28 @@ out: pipe_config->gmch_pfit.lvds_border_bits = border; } -static int i915_panel_invert_brightness; -MODULE_PARM_DESC(invert_brightness, "Invert backlight brightness " - "(-1 force normal, 0 machine defaults, 1 force inversion), please " - "report PCI device ID, subsystem vendor and subsystem device ID " - "to dri-devel@lists.freedesktop.org, if your machine needs it. " - "It will then be included in an upcoming module version."); -module_param_named(invert_brightness, i915_panel_invert_brightness, int, 0600); +enum drm_connector_status +intel_panel_detect(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + /* Assume that the BIOS does not lie through the OpRegion... */ + if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) { + return ioread32(dev_priv->opregion.lid_state) & 0x1 ? + connector_status_connected : + connector_status_disconnected; + } + + switch (i915.panel_ignore_lid) { + case -2: + return connector_status_connected; + case -1: + return connector_status_disconnected; + default: + return connector_status_unknown; + } +} + static u32 intel_panel_compute_brightness(struct intel_connector *connector, u32 val) { @@ -341,10 +407,10 @@ static u32 intel_panel_compute_brightness(struct intel_connector *connector, WARN_ON(panel->backlight.max == 0); - if (i915_panel_invert_brightness < 0) + if (i915.invert_brightness < 0) return val; - if (i915_panel_invert_brightness > 0 || + if (i915.invert_brightness > 0 || dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) { return panel->backlight.max - val; } @@ -501,6 +567,7 @@ void intel_panel_set_backlight(struct intel_connector *connector, u32 level, enum pipe pipe = intel_get_pipe_from_connector(connector); u32 freq; unsigned long flags; + u64 n; if (!panel->backlight.present || pipe == INVALID_PIPE) return; @@ -511,10 +578,9 @@ void intel_panel_set_backlight(struct intel_connector *connector, u32 level, /* scale to hardware max, but be careful to not overflow */ freq = panel->backlight.max; - if (freq < max) - level = level * freq / max; - else - level = freq / max * level; + n = (u64)level * freq; + do_div(n, max); + level = n; panel->backlight.level = level; if (panel->backlight.device) @@ -698,7 +764,7 @@ static void i9xx_enable_backlight(struct intel_connector *connector) freq /= 0xff; ctl = freq << 17; - if (IS_GEN2(dev) && panel->backlight.combination_mode) + if (panel->backlight.combination_mode) ctl |= BLM_LEGACY_MODE; if (IS_PINEVIEW(dev) && panel->backlight.active_low_pwm) ctl |= BLM_POLARITY_PNV; @@ -732,9 +798,6 @@ static void i965_enable_backlight(struct intel_connector *connector) ctl = freq << 16; I915_WRITE(BLC_PWM_CTL, ctl); - /* XXX: combine this into above write? */ - intel_panel_actually_set_backlight(connector, panel->backlight.level); - ctl2 = BLM_PIPE(pipe); if (panel->backlight.combination_mode) ctl2 |= BLM_COMBINATION_MODE; @@ -743,6 +806,8 @@ static void i965_enable_backlight(struct intel_connector *connector) I915_WRITE(BLC_PWM_CTL2, ctl2); POSTING_READ(BLC_PWM_CTL2); I915_WRITE(BLC_PWM_CTL2, ctl2 | BLM_PWM_ENABLE); + + intel_panel_actually_set_backlight(connector, panel->backlight.level); } static void vlv_enable_backlight(struct intel_connector *connector) @@ -804,40 +869,18 @@ void intel_panel_enable_backlight(struct intel_connector *connector) spin_unlock_irqrestore(&dev_priv->backlight_lock, flags); } -enum drm_connector_status -intel_panel_detect(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - /* Assume that the BIOS does not lie through the OpRegion... */ - if (!i915_panel_ignore_lid && dev_priv->opregion.lid_state) { - return ioread32(dev_priv->opregion.lid_state) & 0x1 ? - connector_status_connected : - connector_status_disconnected; - } - - switch (i915_panel_ignore_lid) { - case -2: - return connector_status_connected; - case -1: - return connector_status_disconnected; - default: - return connector_status_unknown; - } -} - #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) static int intel_backlight_device_update_status(struct backlight_device *bd) { struct intel_connector *connector = bl_get_data(bd); struct drm_device *dev = connector->base.dev; - mutex_lock(&dev->mode_config.mutex); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); DRM_DEBUG_KMS("updating intel_backlight, brightness=%d/%d\n", bd->props.brightness, bd->props.max_brightness); intel_panel_set_backlight(connector, bd->props.brightness, bd->props.max_brightness); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_unlock(&dev->mode_config.connection_mutex); return 0; } @@ -849,9 +892,9 @@ static int intel_backlight_device_get_brightness(struct backlight_device *bd) int ret; intel_runtime_pm_get(dev_priv); - mutex_lock(&dev->mode_config.mutex); + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); ret = intel_panel_get_backlight(connector); - mutex_unlock(&dev->mode_config.mutex); + drm_modeset_unlock(&dev->mode_config.connection_mutex); intel_runtime_pm_put(dev_priv); return ret; @@ -979,7 +1022,7 @@ static int i9xx_setup_backlight(struct intel_connector *connector) ctl = I915_READ(BLC_PWM_CTL); - if (IS_GEN2(dev)) + if (IS_GEN2(dev) || IS_I915GM(dev) || IS_I945GM(dev)) panel->backlight.combination_mode = ctl & BLM_LEGACY_MODE; if (IS_PINEVIEW(dev)) @@ -1074,6 +1117,15 @@ int intel_panel_setup_backlight(struct drm_connector *connector) unsigned long flags; int ret; + if (!dev_priv->vbt.backlight.present) { + if (dev_priv->quirks & QUIRK_BACKLIGHT_PRESENT) { + DRM_DEBUG_KMS("no backlight present per VBT, but present per quirk\n"); + } else { + DRM_DEBUG_KMS("no backlight present per VBT\n"); + return 0; + } + } + /* set level and max in panel struct */ spin_lock_irqsave(&dev_priv->backlight_lock, flags); ret = dev_priv->display.setup_backlight(intel_connector); @@ -1081,7 +1133,7 @@ int intel_panel_setup_backlight(struct drm_connector *connector) if (ret) { DRM_DEBUG_KMS("failed to setup backlight for connector %s\n", - drm_get_connector_name(connector)); + connector->name); return ret; } @@ -1107,59 +1159,6 @@ void intel_panel_destroy_backlight(struct drm_connector *connector) intel_backlight_device_unregister(intel_connector); } -/** - * intel_find_panel_downclock - find the reduced downclock for LVDS in EDID - * @dev: drm device - * @fixed_mode : panel native mode - * @connector: LVDS/eDP connector - * - * Return downclock_avail - * Find the reduced downclock for LVDS/eDP in EDID. - */ -struct drm_display_mode * -intel_find_panel_downclock(struct drm_device *dev, - struct drm_display_mode *fixed_mode, - struct drm_connector *connector) -{ - struct drm_display_mode *scan, *tmp_mode; - int temp_downclock; - - temp_downclock = fixed_mode->clock; - tmp_mode = NULL; - - list_for_each_entry(scan, &connector->probed_modes, head) { - /* - * If one mode has the same resolution with the fixed_panel - * mode while they have the different refresh rate, it means - * that the reduced downclock is found. In such - * case we can set the different FPx0/1 to dynamically select - * between low and high frequency. - */ - if (scan->hdisplay == fixed_mode->hdisplay && - scan->hsync_start == fixed_mode->hsync_start && - scan->hsync_end == fixed_mode->hsync_end && - scan->htotal == fixed_mode->htotal && - scan->vdisplay == fixed_mode->vdisplay && - scan->vsync_start == fixed_mode->vsync_start && - scan->vsync_end == fixed_mode->vsync_end && - scan->vtotal == fixed_mode->vtotal) { - if (scan->clock < temp_downclock) { - /* - * The downclock is already found. But we - * expect to find the lower downclock. - */ - temp_downclock = scan->clock; - tmp_mode = scan; - } - } - } - - if (temp_downclock < fixed_mode->clock) - return drm_mode_duplicate(dev, tmp_mode); - else - return NULL; -} - /* Set up chip specific backlight functions */ void intel_panel_init_backlight_funcs(struct drm_device *dev) { @@ -1199,9 +1198,11 @@ void intel_panel_init_backlight_funcs(struct drm_device *dev) } int intel_panel_init(struct intel_panel *panel, - struct drm_display_mode *fixed_mode) + struct drm_display_mode *fixed_mode, + struct drm_display_mode *downclock_mode) { panel->fixed_mode = fixed_mode; + panel->downclock_mode = downclock_mode; return 0; } diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index d77cc81900f..ee72807069e 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -92,12 +92,12 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int cfb_pitch; - int plane, i; + int i; u32 fbc_ctl; cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE; @@ -109,7 +109,6 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) cfb_pitch = (cfb_pitch / 32) - 1; else cfb_pitch = (cfb_pitch / 64) - 1; - plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB; /* Clear old tags */ for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++) @@ -120,7 +119,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) /* Set it up... */ fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE; - fbc_ctl2 |= plane; + fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane); I915_WRITE(FBC_CONTROL2, fbc_ctl2); I915_WRITE(FBC_FENCE_OFF, crtc->y); } @@ -135,7 +134,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc) fbc_ctl |= obj->fence_reg; I915_WRITE(FBC_CONTROL, fbc_ctl); - DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ", + DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n", cfb_pitch, crtc->y, plane_name(intel_crtc->plane)); } @@ -150,21 +149,23 @@ static void g4x_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; u32 dpfc_ctl; - dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN; + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg; - I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY); I915_WRITE(DPFC_FENCE_YOFF, crtc->y); /* enable it... */ - I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN); + I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane)); } @@ -220,22 +221,20 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; u32 dpfc_ctl; - dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); - dpfc_ctl &= DPFC_RESERVED; - dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X); - /* Set persistent mode for front-buffer rendering, ala X. */ - dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE; + dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane); + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; dpfc_ctl |= DPFC_CTL_FENCE_EN; if (IS_GEN5(dev)) dpfc_ctl |= obj->fence_reg; - I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY); I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID); @@ -278,24 +277,31 @@ static void gen7_enable_fbc(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_framebuffer *fb = crtc->fb; + struct drm_framebuffer *fb = crtc->primary->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + u32 dpfc_ctl; - I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj)); + dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane); + if (drm_format_plane_cpp(fb->pixel_format, 0) == 2) + dpfc_ctl |= DPFC_CTL_LIMIT_2X; + else + dpfc_ctl |= DPFC_CTL_LIMIT_1X; + dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN; - I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X | - IVB_DPFC_CTL_FENCE_EN | - intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT); + I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); if (IS_IVYBRIDGE(dev)) { /* WaFbcAsynchFlipDisableFbcQueue:ivb */ - I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS); + I915_WRITE(ILK_DISPLAY_CHICKEN1, + I915_READ(ILK_DISPLAY_CHICKEN1) | + ILK_FBCQ_DIS); } else { - /* WaFbcAsynchFlipDisableFbcQueue:hsw */ - I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe), - HSW_BYPASS_FBC_QUEUE); + /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */ + I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe), + I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) | + HSW_FBCQ_DIS); } I915_WRITE(SNB_DPFC_CTL_SA, @@ -330,11 +336,11 @@ static void intel_fbc_work_fn(struct work_struct *__work) /* Double check that we haven't switched fb without cancelling * the prior work. */ - if (work->crtc->fb == work->fb) { + if (work->crtc->primary->fb == work->fb) { dev_priv->display.enable_fbc(work->crtc); dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane; - dev_priv->fbc.fb_id = work->crtc->fb->base.id; + dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id; dev_priv->fbc.y = work->crtc->y; } @@ -387,7 +393,7 @@ static void intel_enable_fbc(struct drm_crtc *crtc) } work->crtc = crtc; - work->fb = crtc->fb; + work->fb = crtc->primary->fb; INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn); dev_priv->fbc.fbc_work = work; @@ -466,7 +472,7 @@ void intel_update_fbc(struct drm_device *dev) return; } - if (!i915_powersave) { + if (!i915.powersave) { if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) DRM_DEBUG_KMS("fbc disabled per module param\n"); return; @@ -481,7 +487,7 @@ void intel_update_fbc(struct drm_device *dev) * - new fb is too large to fit in compressed buffer * - going to an unsupported config (interlace, pixel multiply, etc.) */ - list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) { + for_each_crtc(dev, tmp_crtc) { if (intel_crtc_active(tmp_crtc) && to_intel_crtc(tmp_crtc)->primary_enabled) { if (crtc) { @@ -493,25 +499,24 @@ void intel_update_fbc(struct drm_device *dev) } } - if (!crtc || crtc->fb == NULL) { + if (!crtc || crtc->primary->fb == NULL) { if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT)) DRM_DEBUG_KMS("no output, disabling\n"); goto out_disable; } intel_crtc = to_intel_crtc(crtc); - fb = crtc->fb; + fb = crtc->primary->fb; intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; adjusted_mode = &intel_crtc->config.adjusted_mode; - if (i915_enable_fbc < 0 && - INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) { + if (i915.enable_fbc < 0) { if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT)) DRM_DEBUG_KMS("disabled per chip default\n"); goto out_disable; } - if (!i915_enable_fbc) { + if (!i915.enable_fbc) { if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM)) DRM_DEBUG_KMS("fbc disabled per module param\n"); goto out_disable; @@ -537,7 +542,7 @@ void intel_update_fbc(struct drm_device *dev) DRM_DEBUG_KMS("mode too large for compression, disabling\n"); goto out_disable; } - if ((INTEL_INFO(dev)->gen < 4 || IS_HASWELL(dev)) && + if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) && intel_crtc->plane != PLANE_A) { if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE)) DRM_DEBUG_KMS("plane not A, disabling compression\n"); @@ -617,7 +622,7 @@ out_disable: static void i915_pineview_get_mem_freq(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u32 tmp; tmp = I915_READ(CLKCFG); @@ -656,7 +661,7 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev) static void i915_ironlake_get_mem_freq(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; u16 ddrpll, csipll; ddrpll = I915_READ16(DDRMPLL1); @@ -1004,7 +1009,7 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) { struct drm_crtc *crtc, *enabled = NULL; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + for_each_crtc(dev, crtc) { if (intel_crtc_active(crtc)) { if (enabled) return NULL; @@ -1035,7 +1040,7 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) crtc = single_enabled_crtc(dev); if (crtc) { const struct drm_display_mode *adjusted_mode; - int pixel_size = crtc->fb->bits_per_pixel / 8; + int pixel_size = crtc->primary->fb->bits_per_pixel / 8; int clock; adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode; @@ -1115,7 +1120,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - pixel_size = crtc->fb->bits_per_pixel / 8; + pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* Use the small buffer method to calculate plane watermark */ entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000; @@ -1128,9 +1133,9 @@ static bool g4x_compute_wm0(struct drm_device *dev, *plane_wm = display->max_wm; /* Use the large buffer method to calculate cursor watermark */ - line_time_us = ((htotal * 1000) / clock); + line_time_us = max(htotal * 1000 / clock, 1); line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; - entries = line_count * 64 * pixel_size; + entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size; tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; if (tlb_miss > 0) entries += tlb_miss; @@ -1202,9 +1207,9 @@ static bool g4x_compute_srwm(struct drm_device *dev, clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - pixel_size = crtc->fb->bits_per_pixel / 8; + pixel_size = crtc->primary->fb->bits_per_pixel / 8; - line_time_us = (htotal * 1000) / clock; + line_time_us = max(htotal * 1000 / clock, 1); line_count = (latency_ns / line_time_us + 1000) / 1000; line_size = hdisplay * pixel_size; @@ -1216,7 +1221,7 @@ static bool g4x_compute_srwm(struct drm_device *dev, *display_wm = entries + display->guard_size; /* calculate the self-refresh watermark for display cursor */ - entries = line_count * pixel_size * 64; + entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width; entries = DIV_ROUND_UP(entries, cursor->cacheline_size); *cursor_wm = entries + cursor->guard_size; @@ -1241,7 +1246,7 @@ static bool vlv_compute_drain_latency(struct drm_device *dev, return false; clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock; - pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */ + pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */ entries = (clock / 1000) * pixel_size; *plane_prec_mult = (entries > 256) ? @@ -1433,11 +1438,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w; - int pixel_size = crtc->fb->bits_per_pixel / 8; + int pixel_size = crtc->primary->fb->bits_per_pixel / 8; unsigned long line_time_us; int entries; - line_time_us = ((htotal * 1000) / clock); + line_time_us = max(htotal * 1000 / clock, 1); /* Use ns/us then divide to preserve precision */ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * @@ -1451,7 +1456,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) entries, srwm); entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * - pixel_size * 64; + pixel_size * to_intel_crtc(crtc)->cursor_width; entries = DIV_ROUND_UP(entries, i965_cursor_wm_info.cacheline_size); cursor_sr = i965_cursor_wm_info.fifo_size - @@ -1506,7 +1511,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) crtc = intel_get_crtc_for_plane(dev, 0); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode; - int cpp = crtc->fb->bits_per_pixel / 8; + int cpp = crtc->primary->fb->bits_per_pixel / 8; if (IS_GEN2(dev)) cpp = 4; @@ -1522,7 +1527,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) crtc = intel_get_crtc_for_plane(dev, 1); if (intel_crtc_active(crtc)) { const struct drm_display_mode *adjusted_mode; - int cpp = crtc->fb->bits_per_pixel / 8; + int cpp = crtc->primary->fb->bits_per_pixel / 8; if (IS_GEN2(dev)) cpp = 4; @@ -1539,6 +1544,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm); + if (IS_I915GM(dev) && enabled) { + struct intel_framebuffer *fb; + + fb = to_intel_framebuffer(enabled->primary->fb); + + /* self-refresh seems busted with untiled */ + if (fb->obj->tiling_mode == I915_TILING_NONE) + enabled = NULL; + } + /* * Overlay gets an aggressive default since video jitter is bad. */ @@ -1559,11 +1574,11 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w; - int pixel_size = enabled->fb->bits_per_pixel / 8; + int pixel_size = enabled->primary->fb->bits_per_pixel / 8; unsigned long line_time_us; int entries; - line_time_us = (htotal * 1000) / clock; + line_time_us = max(htotal * 1000 / clock, 1); /* Use ns/us then divide to preserve precision */ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * @@ -1815,6 +1830,40 @@ static unsigned int ilk_display_fifo_size(const struct drm_device *dev) return 512; } +static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev, + int level, bool is_sprite) +{ + if (INTEL_INFO(dev)->gen >= 8) + /* BDW primary/sprite plane watermarks */ + return level == 0 ? 255 : 2047; + else if (INTEL_INFO(dev)->gen >= 7) + /* IVB/HSW primary/sprite plane watermarks */ + return level == 0 ? 127 : 1023; + else if (!is_sprite) + /* ILK/SNB primary plane watermarks */ + return level == 0 ? 127 : 511; + else + /* ILK/SNB sprite plane watermarks */ + return level == 0 ? 63 : 255; +} + +static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev, + int level) +{ + if (INTEL_INFO(dev)->gen >= 7) + return level == 0 ? 63 : 255; + else + return level == 0 ? 31 : 63; +} + +static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev) +{ + if (INTEL_INFO(dev)->gen >= 8) + return 31; + else + return 15; +} + /* Calculate the maximum primary/sprite plane watermark */ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, int level, @@ -1823,7 +1872,6 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, bool is_sprite) { unsigned int fifo_size = ilk_display_fifo_size(dev); - unsigned int max; /* if sprites aren't enabled, sprites get nothing */ if (is_sprite && !config->sprites_enabled) @@ -1854,19 +1902,7 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev, } /* clamp to max that the registers can hold */ - if (INTEL_INFO(dev)->gen >= 8) - max = level == 0 ? 255 : 2047; - else if (INTEL_INFO(dev)->gen >= 7) - /* IVB/HSW primary/sprite plane watermarks */ - max = level == 0 ? 127 : 1023; - else if (!is_sprite) - /* ILK/SNB primary plane watermarks */ - max = level == 0 ? 127 : 511; - else - /* ILK/SNB sprite plane watermarks */ - max = level == 0 ? 63 : 255; - - return min(fifo_size, max); + return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite)); } /* Calculate the maximum cursor plane watermark */ @@ -1879,23 +1915,10 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev, return 64; /* otherwise just report max that registers can hold */ - if (INTEL_INFO(dev)->gen >= 7) - return level == 0 ? 63 : 255; - else - return level == 0 ? 31 : 63; -} - -/* Calculate the maximum FBC watermark */ -static unsigned int ilk_fbc_wm_max(struct drm_device *dev) -{ - /* max that registers can hold */ - if (INTEL_INFO(dev)->gen >= 8) - return 31; - else - return 15; + return ilk_cursor_wm_reg_max(dev, level); } -static void ilk_compute_wm_maximums(struct drm_device *dev, +static void ilk_compute_wm_maximums(const struct drm_device *dev, int level, const struct intel_wm_config *config, enum intel_ddb_partitioning ddb_partitioning, @@ -1904,7 +1927,17 @@ static void ilk_compute_wm_maximums(struct drm_device *dev, max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); max->cur = ilk_cursor_wm_max(dev, level, config); - max->fbc = ilk_fbc_wm_max(dev); + max->fbc = ilk_fbc_wm_reg_max(dev); +} + +static void ilk_compute_wm_reg_maximums(struct drm_device *dev, + int level, + struct ilk_wm_maximums *max) +{ + max->pri = ilk_plane_wm_reg_max(dev, level, false); + max->spr = ilk_plane_wm_reg_max(dev, level, true); + max->cur = ilk_cursor_wm_reg_max(dev, level); + max->fbc = ilk_fbc_wm_reg_max(dev); } static bool ilk_validate_wm_level(int level, @@ -1948,7 +1981,7 @@ static bool ilk_validate_wm_level(int level, return ret; } -static void ilk_compute_wm_level(struct drm_i915_private *dev_priv, +static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, int level, const struct ilk_pipe_wm_parameters *p, struct intel_wm_level *result) @@ -2043,7 +2076,7 @@ static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5]) wm[3] *= 2; } -static int ilk_wm_max_level(const struct drm_device *dev) +int ilk_wm_max_level(const struct drm_device *dev) { /* how many WM levels are we expecting */ if (IS_HASWELL(dev) || IS_BROADWELL(dev)) @@ -2079,7 +2112,44 @@ static void intel_print_wm_latency(struct drm_device *dev, } } -static void intel_setup_wm_latency(struct drm_device *dev) +static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, + uint16_t wm[5], uint16_t min) +{ + int level, max_level = ilk_wm_max_level(dev_priv->dev); + + if (wm[0] >= min) + return false; + + wm[0] = max(wm[0], min); + for (level = 1; level <= max_level; level++) + wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5)); + + return true; +} + +static void snb_wm_latency_quirk(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + bool changed; + + /* + * The BIOS provided WM memory latency values are often + * inadequate for high resolution displays. Adjust them. + */ + changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) | + ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) | + ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); + + if (!changed) + return; + + DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n"); + intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); + intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); + intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); +} + +static void ilk_setup_wm_latency(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -2096,41 +2166,58 @@ static void intel_setup_wm_latency(struct drm_device *dev) intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency); intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency); intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency); + + if (IS_GEN6(dev)) + snb_wm_latency_quirk(dev); } static void ilk_compute_wm_parameters(struct drm_crtc *crtc, - struct ilk_pipe_wm_parameters *p, - struct intel_wm_config *config) + struct ilk_pipe_wm_parameters *p) { struct drm_device *dev = crtc->dev; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; struct drm_plane *plane; - p->active = intel_crtc_active(crtc); - if (p->active) { - p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal; - p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); - p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8; - p->cur.bytes_per_pixel = 4; - p->pri.horiz_pixels = intel_crtc->config.pipe_src_w; - p->cur.horiz_pixels = 64; - /* TODO: for now, assume primary and cursor planes are always enabled. */ - p->pri.enabled = true; - p->cur.enabled = true; - } - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) - config->num_pipes_active += intel_crtc_active(crtc); + if (!intel_crtc_active(crtc)) + return; - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { + p->active = true; + p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal; + p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc); + p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8; + p->cur.bytes_per_pixel = 4; + p->pri.horiz_pixels = intel_crtc->config.pipe_src_w; + p->cur.horiz_pixels = intel_crtc->cursor_width; + /* TODO: for now, assume primary and cursor planes are always enabled. */ + p->pri.enabled = true; + p->cur.enabled = true; + + drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) { struct intel_plane *intel_plane = to_intel_plane(plane); - if (intel_plane->pipe == pipe) + if (intel_plane->pipe == pipe) { p->spr = intel_plane->wm; + break; + } + } +} + +static void ilk_compute_wm_config(struct drm_device *dev, + struct intel_wm_config *config) +{ + struct intel_crtc *intel_crtc; - config->sprites_enabled |= intel_plane->wm.enabled; - config->sprites_scaled |= intel_plane->wm.scaled; + /* Compute the currently _active_ config */ + for_each_intel_crtc(dev, intel_crtc) { + const struct intel_pipe_wm *wm = &intel_crtc->wm.active; + + if (!wm->pipe_enabled) + continue; + + config->sprites_enabled |= wm->sprites_enabled; + config->sprites_scaled |= wm->sprites_scaled; + config->num_pipes_active++; } } @@ -2140,7 +2227,7 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, struct intel_pipe_wm *pipe_wm) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + const struct drm_i915_private *dev_priv = dev->dev_private; int level, max_level = ilk_wm_max_level(dev); /* LP0 watermark maximums depend on this pipe alone */ struct intel_wm_config config = { @@ -2150,8 +2237,9 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, }; struct ilk_wm_maximums max; - /* LP0 watermarks always use 1/2 DDB partitioning */ - ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); + pipe_wm->pipe_enabled = params->active; + pipe_wm->sprites_enabled = params->spr.enabled; + pipe_wm->sprites_scaled = params->spr.scaled; /* ILK/SNB: LP2+ watermarks only w/o sprites */ if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled) @@ -2161,15 +2249,37 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc, if (params->spr.scaled) max_level = 0; - for (level = 0; level <= max_level; level++) - ilk_compute_wm_level(dev_priv, level, params, - &pipe_wm->wm[level]); + ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]); if (IS_HASWELL(dev) || IS_BROADWELL(dev)) pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc); + /* LP0 watermarks always use 1/2 DDB partitioning */ + ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); + /* At least LP0 must be valid */ - return ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]); + if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) + return false; + + ilk_compute_wm_reg_maximums(dev, 1, &max); + + for (level = 1; level <= max_level; level++) { + struct intel_wm_level wm = {}; + + ilk_compute_wm_level(dev_priv, level, params, &wm); + + /* + * Disable any watermark level that exceeds the + * register maximums since such watermarks are + * always invalid. + */ + if (!ilk_validate_wm_level(level, &max, &wm)) + break; + + pipe_wm->wm[level] = wm; + } + + return true; } /* @@ -2181,20 +2291,28 @@ static void ilk_merge_wm_level(struct drm_device *dev, { const struct intel_crtc *intel_crtc; - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) { - const struct intel_wm_level *wm = - &intel_crtc->wm.active.wm[level]; + ret_wm->enable = true; + + for_each_intel_crtc(dev, intel_crtc) { + const struct intel_pipe_wm *active = &intel_crtc->wm.active; + const struct intel_wm_level *wm = &active->wm[level]; + + if (!active->pipe_enabled) + continue; + /* + * The watermark values may have been used in the past, + * so we must maintain them in the registers for some + * time even if the level is now disabled. + */ if (!wm->enable) - return; + ret_wm->enable = false; ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val); ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val); ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val); ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val); } - - ret_wm->enable = true; } /* @@ -2206,6 +2324,7 @@ static void ilk_wm_merge(struct drm_device *dev, struct intel_pipe_wm *merged) { int level, max_level = ilk_wm_max_level(dev); + int last_enabled_level = max_level; /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) && @@ -2221,15 +2340,19 @@ static void ilk_wm_merge(struct drm_device *dev, ilk_merge_wm_level(dev, level, wm); - if (!ilk_validate_wm_level(level, max, wm)) - break; + if (level > last_enabled_level) + wm->enable = false; + else if (!ilk_validate_wm_level(level, max, wm)) + /* make sure all following levels get disabled */ + last_enabled_level = level - 1; /* * The spec says it is preferred to disable * FBC WMs instead of disabling a WM level. */ if (wm->fbc_val > max->fbc) { - merged->fbc_wm_enabled = false; + if (wm->enable) + merged->fbc_wm_enabled = false; wm->fbc_val = 0; } } @@ -2284,14 +2407,19 @@ static void ilk_compute_wm_results(struct drm_device *dev, level = ilk_wm_lp_to_level(wm_lp, merged); r = &merged->wm[level]; - if (!r->enable) - break; - results->wm_lp[wm_lp - 1] = WM3_LP_EN | + /* + * Maintain the watermark values even if the level is + * disabled. Doing otherwise could cause underruns. + */ + results->wm_lp[wm_lp - 1] = (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) | (r->pri_val << WM1_LP_SR_SHIFT) | r->cur_val; + if (r->enable) + results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; + if (INTEL_INFO(dev)->gen >= 8) results->wm_lp[wm_lp - 1] |= r->fbc_val << WM1_LP_FBC_SHIFT_BDW; @@ -2299,6 +2427,10 @@ static void ilk_compute_wm_results(struct drm_device *dev, results->wm_lp[wm_lp - 1] |= r->fbc_val << WM1_LP_FBC_SHIFT; + /* + * Always set WM1S_LP_EN when spr_val != 0, even if the + * level is disabled. Doing otherwise could cause underruns. + */ if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) { WARN_ON(wm_lp != 1); results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; @@ -2307,7 +2439,7 @@ static void ilk_compute_wm_results(struct drm_device *dev, } /* LP0 register values */ - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) { + for_each_intel_crtc(dev, intel_crtc) { enum pipe pipe = intel_crtc->pipe; const struct intel_wm_level *r = &intel_crtc->wm.active.wm[0]; @@ -2542,7 +2674,7 @@ static void ilk_update_wm(struct drm_crtc *crtc) struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; struct intel_wm_config config = {}; - ilk_compute_wm_parameters(crtc, ¶ms, &config); + ilk_compute_wm_parameters(crtc, ¶ms); intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm); @@ -2551,6 +2683,8 @@ static void ilk_update_wm(struct drm_crtc *crtc) intel_crtc->wm.active = pipe_wm; + ilk_compute_wm_config(dev, &config); + ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max); ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); @@ -2617,7 +2751,9 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) if (IS_HASWELL(dev) || IS_BROADWELL(dev)) hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe)); - if (intel_crtc_active(crtc)) { + active->pipe_enabled = intel_crtc_active(crtc); + + if (active->pipe_enabled) { u32 tmp = hw->wm_pipe[pipe]; /* @@ -2650,7 +2786,7 @@ void ilk_wm_get_hw_state(struct drm_device *dev) struct ilk_wm_values *hw = &dev_priv->wm.hw; struct drm_crtc *crtc; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + for_each_crtc(dev, crtc) ilk_pipe_wm_get_hw_state(crtc); hw->wm_lp[0] = I915_READ(WM1_LP_ILK); @@ -2658,8 +2794,10 @@ void ilk_wm_get_hw_state(struct drm_device *dev) hw->wm_lp[2] = I915_READ(WM3_LP_ILK); hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); - hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); - hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); + if (INTEL_INFO(dev)->gen >= 7) { + hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); + hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); + } if (IS_HASWELL(dev) || IS_BROADWELL(dev)) hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ? @@ -2738,7 +2876,7 @@ intel_alloc_context_page(struct drm_device *dev) return NULL; } - ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false); + ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0); if (ret) { DRM_ERROR("failed to pin power context: %d\n", ret); goto err_unref; @@ -2753,7 +2891,7 @@ intel_alloc_context_page(struct drm_device *dev) return ctx; err_unpin: - i915_gem_object_unpin(ctx); + i915_gem_object_ggtt_unpin(ctx); err_unref: drm_gem_object_unreference(&ctx->base); return NULL; @@ -2901,9 +3039,9 @@ static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val) * the hw runs at the minimal clock before selecting the desired * frequency, if the down threshold expires in that window we will not * receive a down interrupt. */ - limits = dev_priv->rps.max_delay << 24; - if (val <= dev_priv->rps.min_delay) - limits |= dev_priv->rps.min_delay << 16; + limits = dev_priv->rps.max_freq_softlimit << 24; + if (val <= dev_priv->rps.min_freq_softlimit) + limits |= dev_priv->rps.min_freq_softlimit << 16; return limits; } @@ -2915,26 +3053,26 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) new_power = dev_priv->rps.power; switch (dev_priv->rps.power) { case LOW_POWER: - if (val > dev_priv->rps.rpe_delay + 1 && val > dev_priv->rps.cur_delay) + if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) new_power = BETWEEN; break; case BETWEEN: - if (val <= dev_priv->rps.rpe_delay && val < dev_priv->rps.cur_delay) + if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) new_power = LOW_POWER; - else if (val >= dev_priv->rps.rp0_delay && val > dev_priv->rps.cur_delay) + else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) new_power = HIGH_POWER; break; case HIGH_POWER: - if (val < (dev_priv->rps.rp1_delay + dev_priv->rps.rp0_delay) >> 1 && val < dev_priv->rps.cur_delay) + if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) new_power = BETWEEN; break; } /* Max/min bins are special */ - if (val == dev_priv->rps.min_delay) + if (val == dev_priv->rps.min_freq_softlimit) new_power = LOW_POWER; - if (val == dev_priv->rps.max_delay) + if (val == dev_priv->rps.max_freq_softlimit) new_power = HIGH_POWER; if (new_power == dev_priv->rps.power) return; @@ -3000,41 +3138,112 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) dev_priv->rps.last_adj = 0; } +static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) +{ + u32 mask = 0; + + if (val > dev_priv->rps.min_freq_softlimit) + mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT; + if (val < dev_priv->rps.max_freq_softlimit) + mask |= GEN6_PM_RP_UP_THRESHOLD; + + /* IVB and SNB hard hangs on looping batchbuffer + * if GEN6_PM_UP_EI_EXPIRED is masked. + */ + if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev)) + mask |= GEN6_PM_RP_UP_EI_EXPIRED; + + if (IS_GEN8(dev_priv->dev)) + mask |= GEN8_PMINTR_REDIRECT_TO_NON_DISP; + + return ~mask; +} + +/* gen6_set_rps is called to update the frequency request, but should also be + * called when the range (min_delay and max_delay) is modified so that we can + * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ void gen6_set_rps(struct drm_device *dev, u8 val) { struct drm_i915_private *dev_priv = dev->dev_private; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - WARN_ON(val > dev_priv->rps.max_delay); - WARN_ON(val < dev_priv->rps.min_delay); - - if (val == dev_priv->rps.cur_delay) - return; + WARN_ON(val > dev_priv->rps.max_freq_softlimit); + WARN_ON(val < dev_priv->rps.min_freq_softlimit); - gen6_set_rps_thresholds(dev_priv, val); + /* min/max delay may still have been modified so be sure to + * write the limits value. + */ + if (val != dev_priv->rps.cur_freq) { + gen6_set_rps_thresholds(dev_priv, val); - if (IS_HASWELL(dev)) - I915_WRITE(GEN6_RPNSWREQ, - HSW_FREQUENCY(val)); - else - I915_WRITE(GEN6_RPNSWREQ, - GEN6_FREQUENCY(val) | - GEN6_OFFSET(0) | - GEN6_AGGRESSIVE_TURBO); + if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + I915_WRITE(GEN6_RPNSWREQ, + HSW_FREQUENCY(val)); + else + I915_WRITE(GEN6_RPNSWREQ, + GEN6_FREQUENCY(val) | + GEN6_OFFSET(0) | + GEN6_AGGRESSIVE_TURBO); + } /* Make sure we continue to get interrupts * until we hit the minimum or maximum frequencies. */ - I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - gen6_rps_limits(dev_priv, val)); + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val)); + I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); POSTING_READ(GEN6_RPNSWREQ); - dev_priv->rps.cur_delay = val; - + dev_priv->rps.cur_freq = val; trace_intel_gpu_freq_change(val * 50); } +/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down + * + * * If Gfx is Idle, then + * 1. Mask Turbo interrupts + * 2. Bring up Gfx clock + * 3. Change the freq to Rpn and wait till P-Unit updates freq + * 4. Clear the Force GFX CLK ON bit so that Gfx can down + * 5. Unmask Turbo interrupts +*/ +static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + + /* Latest VLV doesn't need to force the gfx clock */ + if (dev->pdev->revision >= 0xd) { + valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); + return; + } + + /* + * When we are idle. Drop to min voltage state. + */ + + if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit) + return; + + /* Mask turbo interrupt so that they will not come in between */ + I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); + + vlv_force_gfx_clock(dev_priv, true); + + dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit; + + vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, + dev_priv->rps.min_freq_softlimit); + + if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) + & GENFREQSTATUS) == 0, 5)) + DRM_ERROR("timed out waiting for Punit\n"); + + vlv_force_gfx_clock(dev_priv, false); + + I915_WRITE(GEN6_PMINTRMSK, + gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); +} + void gen6_rps_idle(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; @@ -3042,9 +3251,9 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { if (IS_VALLEYVIEW(dev)) - valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + vlv_set_rps_idle(dev_priv); else - gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3057,9 +3266,9 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv) mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { if (IS_VALLEYVIEW(dev)) - valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay); + valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); else - gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay); + gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit); dev_priv->rps.last_adj = 0; } mutex_unlock(&dev_priv->rps.hw_lock); @@ -3070,30 +3279,50 @@ void valleyview_set_rps(struct drm_device *dev, u8 val) struct drm_i915_private *dev_priv = dev->dev_private; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - WARN_ON(val > dev_priv->rps.max_delay); - WARN_ON(val < dev_priv->rps.min_delay); + WARN_ON(val > dev_priv->rps.max_freq_softlimit); + WARN_ON(val < dev_priv->rps.min_freq_softlimit); DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay), - dev_priv->rps.cur_delay, + vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), + dev_priv->rps.cur_freq, vlv_gpu_freq(dev_priv, val), val); - if (val == dev_priv->rps.cur_delay) - return; - - vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); + if (val != dev_priv->rps.cur_freq) + vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val); - dev_priv->rps.cur_delay = val; + I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val)); + dev_priv->rps.cur_freq = val; trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val)); } +static void gen8_disable_rps_interrupts(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); + I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) & + ~dev_priv->pm_rps_events); + /* Complete PM interrupt masking here doesn't race with the rps work + * item again unmasking PM interrupts because that is using a different + * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in + * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which + * gen8_enable_rps will clean up. */ + + spin_lock_irq(&dev_priv->irq_lock); + dev_priv->rps.pm_iir = 0; + spin_unlock_irq(&dev_priv->irq_lock); + + I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); +} + static void gen6_disable_rps_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); - I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS); + I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & + ~dev_priv->pm_rps_events); /* Complete PM interrupt masking here doesn't race with the rps work * item again unmasking PM interrupts because that is using a different * register (PMIMR) to mask PM interrupts. The only risk is in leaving @@ -3103,7 +3332,7 @@ static void gen6_disable_rps_interrupts(struct drm_device *dev) dev_priv->rps.pm_iir = 0; spin_unlock_irq(&dev_priv->irq_lock); - I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS); + I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events); } static void gen6_disable_rps(struct drm_device *dev) @@ -3113,7 +3342,10 @@ static void gen6_disable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_CONTROL, 0); I915_WRITE(GEN6_RPNSWREQ, 1 << 31); - gen6_disable_rps_interrupts(dev); + if (IS_BROADWELL(dev)) + gen8_disable_rps_interrupts(dev); + else + gen6_disable_rps_interrupts(dev); } static void valleyview_disable_rps(struct drm_device *dev) @@ -3123,78 +3355,111 @@ static void valleyview_disable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_CONTROL, 0); gen6_disable_rps_interrupts(dev); - - if (dev_priv->vlv_pctx) { - drm_gem_object_unreference(&dev_priv->vlv_pctx->base); - dev_priv->vlv_pctx = NULL; - } } static void intel_print_rc6_info(struct drm_device *dev, u32 mode) { - if (IS_GEN6(dev)) - DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n"); - - if (IS_HASWELL(dev)) - DRM_DEBUG_DRIVER("Haswell: only RC6 available\n"); - + if (IS_VALLEYVIEW(dev)) { + if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1))) + mode = GEN6_RC_CTL_RC6_ENABLE; + else + mode = 0; + } DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n", - (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", - (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", - (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); + (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off", + (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off", + (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off"); } -int intel_enable_rc6(const struct drm_device *dev) +static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) { /* No RC6 before Ironlake */ if (INTEL_INFO(dev)->gen < 5) return 0; + /* RC6 is only on Ironlake mobile not on desktop */ + if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev)) + return 0; + /* Respect the kernel parameter if it is set */ - if (i915_enable_rc6 >= 0) - return i915_enable_rc6; + if (enable_rc6 >= 0) { + int mask; + + if (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev)) + mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE | + INTEL_RC6pp_ENABLE; + else + mask = INTEL_RC6_ENABLE; + + if ((enable_rc6 & mask) != enable_rc6) + DRM_INFO("Adjusting RC6 mask to %d (requested %d, valid %d)\n", + enable_rc6 & mask, enable_rc6, mask); + + return enable_rc6 & mask; + } /* Disable RC6 on Ironlake */ if (INTEL_INFO(dev)->gen == 5) return 0; - if (IS_HASWELL(dev)) - return INTEL_RC6_ENABLE; + if (IS_IVYBRIDGE(dev)) + return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); - /* snb/ivb have more than one rc6 state. */ - if (INTEL_INFO(dev)->gen == 6) - return INTEL_RC6_ENABLE; + return INTEL_RC6_ENABLE; +} - return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); +int intel_enable_rc6(const struct drm_device *dev) +{ + return i915.enable_rc6; +} + +static void gen8_enable_rps_interrupts(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + spin_lock_irq(&dev_priv->irq_lock); + WARN_ON(dev_priv->rps.pm_iir); + bdw_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events); + spin_unlock_irq(&dev_priv->irq_lock); } static void gen6_enable_rps_interrupts(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 enabled_intrs; spin_lock_irq(&dev_priv->irq_lock); WARN_ON(dev_priv->rps.pm_iir); - snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS); - I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS); + snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); +} - /* only unmask PM interrupts we need. Mask all others. */ - enabled_intrs = GEN6_PM_RPS_EVENTS; +static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_cap) +{ + /* All of these values are in units of 50MHz */ + dev_priv->rps.cur_freq = 0; + /* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */ + dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; + dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff; + dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff; + /* XXX: only BYT has a special efficient freq */ + dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; + /* hw_max = RP0 until we check for overclocking */ + dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; - /* IVB and SNB hard hangs on looping batchbuffer - * if GEN6_PM_UP_EI_EXPIRED is masked. - */ - if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) - enabled_intrs |= GEN6_PM_RP_UP_EI_EXPIRED; + /* Preserve min/max settings in case of re-init */ + if (dev_priv->rps.max_freq_softlimit == 0) + dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; - I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs); + if (dev_priv->rps.min_freq_softlimit == 0) + dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; } static void gen8_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; uint32_t rc6_mask = 0, rp_state_cap; int unused; @@ -3209,6 +3474,7 @@ static void gen8_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_CONTROL, 0); rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + parse_rp_state_cap(dev_priv, rp_state_cap); /* 2b: Program RC6 thresholds.*/ I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); @@ -3222,21 +3488,23 @@ static void gen8_enable_rps(struct drm_device *dev) /* 3: Enable RC6 */ if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) rc6_mask = GEN6_RC_CTL_RC6_ENABLE; - DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off"); + intel_print_rc6_info(dev, rc6_mask); I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | - GEN6_RC_CTL_EI_MODE(1) | - rc6_mask); + GEN6_RC_CTL_EI_MODE(1) | + rc6_mask); /* 4 Program defaults and thresholds for RPS*/ - I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */ - I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(12)); /* Request 600 MHz */ + I915_WRITE(GEN6_RPNSWREQ, + HSW_FREQUENCY(dev_priv->rps.rp1_freq)); + I915_WRITE(GEN6_RC_VIDEO_FREQ, + HSW_FREQUENCY(dev_priv->rps.rp1_freq)); /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */ /* Docs recommend 900MHz, and 300 MHz respectively */ I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, - dev_priv->rps.max_delay << 24 | - dev_priv->rps.min_delay << 16); + dev_priv->rps.max_freq_softlimit << 24 | + dev_priv->rps.min_freq_softlimit << 16); I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */ I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/ @@ -3258,7 +3526,7 @@ static void gen8_enable_rps(struct drm_device *dev) gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8); - gen6_enable_rps_interrupts(dev); + gen8_enable_rps_interrupts(dev); gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } @@ -3266,10 +3534,10 @@ static void gen8_enable_rps(struct drm_device *dev) static void gen6_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; u32 rp_state_cap; u32 gt_perf_status; - u32 rc6vids, pcu_mbox, rc6_mask = 0; + u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; u32 gtfifodbg; int rc6_mode; int i, ret; @@ -3295,13 +3563,7 @@ static void gen6_enable_rps(struct drm_device *dev) rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); - /* In units of 50MHz */ - dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff; - dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff; - dev_priv->rps.rp1_delay = (rp_state_cap >> 8) & 0xff; - dev_priv->rps.rp0_delay = (rp_state_cap >> 0) & 0xff; - dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay; - dev_priv->rps.cur_delay = 0; + parse_rp_state_cap(dev_priv, rp_state_cap); /* disable the counters and set deterministic thresholds */ I915_WRITE(GEN6_RC_CONTROL, 0); @@ -3350,21 +3612,19 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0); - if (!ret) { - pcu_mbox = 0; - ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); - if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */ - DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n", - (dev_priv->rps.max_delay & 0xff) * 50, - (pcu_mbox & 0xff) * 50); - dev_priv->rps.hw_max = pcu_mbox & 0xff; - } - } else { + if (ret) DRM_DEBUG_DRIVER("Failed to set the min frequency\n"); + + ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox); + if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */ + DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n", + (dev_priv->rps.max_freq_softlimit & 0xff) * 50, + (pcu_mbox & 0xff) * 50); + dev_priv->rps.max_freq = pcu_mbox & 0xff; } dev_priv->rps.power = HIGH_POWER; /* force a reset */ - gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay); + gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); gen6_enable_rps_interrupts(dev); @@ -3385,7 +3645,7 @@ static void gen6_enable_rps(struct drm_device *dev) gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL); } -void gen6_update_ring_freq(struct drm_device *dev) +static void __gen6_update_ring_freq(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int min_freq = 15; @@ -3420,9 +3680,9 @@ void gen6_update_ring_freq(struct drm_device *dev) * to use for memory access. We do this by specifying the IA frequency * the PCU should use as a reference to determine the ring frequency. */ - for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay; + for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit; gpu_freq--) { - int diff = dev_priv->rps.max_delay - gpu_freq; + int diff = dev_priv->rps.max_freq_softlimit - gpu_freq; unsigned int ia_freq = 0, ring_freq = 0; if (INTEL_INFO(dev)->gen >= 8) { @@ -3455,6 +3715,18 @@ void gen6_update_ring_freq(struct drm_device *dev) } } +void gen6_update_ring_freq(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev)) + return; + + mutex_lock(&dev_priv->rps.hw_lock); + __gen6_update_ring_freq(dev); + mutex_unlock(&dev_priv->rps.hw_lock); +} + int valleyview_rps_max_freq(struct drm_i915_private *dev_priv) { u32 val, rp0; @@ -3485,6 +3757,15 @@ int valleyview_rps_min_freq(struct drm_i915_private *dev_priv) return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff; } +/* Check that the pctx buffer wasn't move under us. */ +static void valleyview_check_pctx(struct drm_i915_private *dev_priv) +{ + unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095; + + WARN_ON(pctx_addr != dev_priv->mm.stolen_base + + dev_priv->vlv_pctx->stolen->start); +} + static void valleyview_setup_pctx(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -3493,6 +3774,8 @@ static void valleyview_setup_pctx(struct drm_device *dev) u32 pcbr; int pctx_size = 24*1024; + WARN_ON(!mutex_is_locked(&dev->struct_mutex)); + pcbr = I915_READ(VLV_PCBR); if (pcbr) { /* BIOS set it up already, grab the pre-alloc'd space */ @@ -3527,23 +3810,73 @@ out: dev_priv->vlv_pctx = pctx; } +static void valleyview_cleanup_pctx(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (WARN_ON(!dev_priv->vlv_pctx)) + return; + + drm_gem_object_unreference(&dev_priv->vlv_pctx->base); + dev_priv->vlv_pctx = NULL; +} + +static void valleyview_init_gt_powersave(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + valleyview_setup_pctx(dev); + + mutex_lock(&dev_priv->rps.hw_lock); + + dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv); + dev_priv->rps.rp0_freq = dev_priv->rps.max_freq; + DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", + vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq), + dev_priv->rps.max_freq); + + dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv); + DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", + vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), + dev_priv->rps.efficient_freq); + + dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv); + DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", + vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq), + dev_priv->rps.min_freq); + + /* Preserve min/max settings in case of re-init */ + if (dev_priv->rps.max_freq_softlimit == 0) + dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; + + if (dev_priv->rps.min_freq_softlimit == 0) + dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq; + + mutex_unlock(&dev_priv->rps.hw_lock); +} + +static void valleyview_cleanup_gt_powersave(struct drm_device *dev) +{ + valleyview_cleanup_pctx(dev); +} + static void valleyview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; u32 gtfifodbg, val, rc6_mode = 0; int i; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); + valleyview_check_pctx(dev_priv); + if ((gtfifodbg = I915_READ(GTFIFODBG))) { DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", gtfifodbg); I915_WRITE(GTFIFODBG, gtfifodbg); } - valleyview_setup_pctx(dev); - /* If VLV, Forcewake all wells, else re-direct to regular path */ gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); @@ -3588,32 +3921,16 @@ static void valleyview_enable_rps(struct drm_device *dev) DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no"); DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); - dev_priv->rps.cur_delay = (val >> 8) & 0xff; + dev_priv->rps.cur_freq = (val >> 8) & 0xff; DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay), - dev_priv->rps.cur_delay); - - dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv); - dev_priv->rps.hw_max = dev_priv->rps.max_delay; - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay), - dev_priv->rps.max_delay); - - dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv); - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay), - dev_priv->rps.rpe_delay); - - dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv); - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay), - dev_priv->rps.min_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq), + dev_priv->rps.cur_freq); DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", - vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay), - dev_priv->rps.rpe_delay); + vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), + dev_priv->rps.efficient_freq); - valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay); + valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); gen6_enable_rps_interrupts(dev); @@ -3625,13 +3942,13 @@ void ironlake_teardown_rc6(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; if (dev_priv->ips.renderctx) { - i915_gem_object_unpin(dev_priv->ips.renderctx); + i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx); drm_gem_object_unreference(&dev_priv->ips.renderctx->base); dev_priv->ips.renderctx = NULL; } if (dev_priv->ips.pwrctx) { - i915_gem_object_unpin(dev_priv->ips.pwrctx); + i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx); drm_gem_object_unreference(&dev_priv->ips.pwrctx->base); dev_priv->ips.pwrctx = NULL; } @@ -3677,7 +3994,7 @@ static int ironlake_setup_rc6(struct drm_device *dev) static void ironlake_enable_rc6(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; bool was_interruptible; int ret; @@ -3735,7 +4052,7 @@ static void ironlake_enable_rc6(struct drm_device *dev) I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) | PWRCTX_EN); I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT); - intel_print_rc6_info(dev, INTEL_RC6_ENABLE); + intel_print_rc6_info(dev, GEN6_RC_CTL_RC6_ENABLE); } static unsigned long intel_pxfreq(u32 vidfreq) @@ -3823,9 +4140,10 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; unsigned long val; - if (dev_priv->info->gen != 5) + if (INTEL_INFO(dev)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -3854,6 +4172,7 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv) static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { + struct drm_device *dev = dev_priv->dev; static const struct v_table { u16 vd; /* in .1 mil */ u16 vm; /* in .1 mil */ @@ -3987,7 +4306,7 @@ static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { 16000, 14875, }, { 16125, 15000, }, }; - if (dev_priv->info->is_mobile) + if (INTEL_INFO(dev)->is_mobile) return v_table[pxvid].vm; else return v_table[pxvid].vd; @@ -4030,7 +4349,9 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) void i915_update_gfx_val(struct drm_i915_private *dev_priv) { - if (dev_priv->info->gen != 5) + struct drm_device *dev = dev_priv->dev; + + if (INTEL_INFO(dev)->gen != 5) return; spin_lock_irq(&mchdev_lock); @@ -4047,7 +4368,7 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) assert_spin_locked(&mchdev_lock); - pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4)); + pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4)); pxvid = (pxvid >> 24) & 0x7f; ext_v = pvid_to_extvid(dev_priv, pxvid); @@ -4079,9 +4400,10 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) { + struct drm_device *dev = dev_priv->dev; unsigned long val; - if (dev_priv->info->gen != 5) + if (INTEL_INFO(dev)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -4184,7 +4506,7 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower); bool i915_gpu_busy(void) { struct drm_i915_private *dev_priv; - struct intel_ring_buffer *ring; + struct intel_engine_cs *ring; bool ret = false; int i; @@ -4270,6 +4592,7 @@ void intel_gpu_ips_teardown(void) i915_mch_dev = NULL; spin_unlock_irq(&mchdev_lock); } + static void intel_init_emon(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4341,6 +4664,20 @@ static void intel_init_emon(struct drm_device *dev) dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); } +void intel_init_gt_powersave(struct drm_device *dev) +{ + i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6); + + if (IS_VALLEYVIEW(dev)) + valleyview_init_gt_powersave(dev); +} + +void intel_cleanup_gt_powersave(struct drm_device *dev) +{ + if (IS_VALLEYVIEW(dev)) + valleyview_cleanup_gt_powersave(dev); +} + void intel_disable_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4351,8 +4688,10 @@ void intel_disable_gt_powersave(struct drm_device *dev) if (IS_IRONLAKE_M(dev)) { ironlake_disable_drps(dev); ironlake_disable_rc6(dev); - } else if (INTEL_INFO(dev)->gen >= 6) { - cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work); + } else if (IS_GEN6(dev) || IS_GEN7(dev) || IS_BROADWELL(dev)) { + if (cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work)) + intel_runtime_pm_put(dev_priv); + cancel_work_sync(&dev_priv->rps.work); mutex_lock(&dev_priv->rps.hw_lock); if (IS_VALLEYVIEW(dev)) @@ -4377,13 +4716,15 @@ static void intel_gen6_powersave_work(struct work_struct *work) valleyview_enable_rps(dev); } else if (IS_BROADWELL(dev)) { gen8_enable_rps(dev); - gen6_update_ring_freq(dev); + __gen6_update_ring_freq(dev); } else { gen6_enable_rps(dev); - gen6_update_ring_freq(dev); + __gen6_update_ring_freq(dev); } dev_priv->rps.enabled = true; mutex_unlock(&dev_priv->rps.hw_lock); + + intel_runtime_pm_put(dev_priv); } void intel_enable_gt_powersave(struct drm_device *dev) @@ -4391,20 +4732,38 @@ void intel_enable_gt_powersave(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; if (IS_IRONLAKE_M(dev)) { + mutex_lock(&dev->struct_mutex); ironlake_enable_drps(dev); ironlake_enable_rc6(dev); intel_init_emon(dev); - } else if (IS_GEN6(dev) || IS_GEN7(dev)) { + mutex_unlock(&dev->struct_mutex); + } else if (IS_GEN6(dev) || IS_GEN7(dev) || IS_BROADWELL(dev)) { /* * PCU communication is slow and this doesn't need to be * done at any specific time, so do this out of our fast path * to make resume and init faster. + * + * We depend on the HW RC6 power context save/restore + * mechanism when entering D3 through runtime PM suspend. So + * disable RPM until RPS/RC6 is properly setup. We can only + * get here via the driver load/system resume/runtime resume + * paths, so the _noresume version is enough (and in case of + * runtime resume it's necessary). */ - schedule_delayed_work(&dev_priv->rps.delayed_resume_work, - round_jiffies_up_relative(HZ)); + if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work, + round_jiffies_up_relative(HZ))) + intel_runtime_pm_get_noresume(dev_priv); } } +void intel_reset_gt_powersave(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + dev_priv->rps.enabled = false; + intel_enable_gt_powersave(dev); +} + static void ibx_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4510,6 +4869,9 @@ static void ironlake_init_clock_gating(struct drm_device *dev) I915_WRITE(CACHE_MODE_0, _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); + /* WaDisable_RenderCache_OperationalFlush:ilk */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); + g4x_disable_trickle_feed(dev); ibx_init_clock_gating(dev); @@ -4585,6 +4947,20 @@ static void gen6_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE)); + /* WaDisable_RenderCache_OperationalFlush:snb */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); + + /* + * BSpec recoomends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN6_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + ilk_init_lp_watermarks(dev); I915_WRITE(CACHE_MODE_0, @@ -4605,17 +4981,24 @@ static void gen6_init_clock_gating(struct drm_device *dev) * According to the spec, bit 11 (RCCUNIT) must also be set, * but we didn't debug actual testcases to find it out. * - * Also apply WaDisableVDSUnitClockGating:snb and - * WaDisableRCPBUnitClockGating:snb. + * WaDisableRCCUnitClockGating:snb + * WaDisableRCPBUnitClockGating:snb */ I915_WRITE(GEN6_UCGCTL2, - GEN7_VDSUNIT_CLOCK_GATE_DISABLE | GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | GEN6_RCCUNIT_CLOCK_GATE_DISABLE); - /* Bspec says we need to always set all mask bits. */ - I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) | - _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL); + /* WaStripsFansDisableFastClipPerformanceFix:snb */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL)); + + /* + * Bspec says: + * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and + * 3DSTATE_SF number of SF output attributes is more than 16." + */ + I915_WRITE(_3D_CHICKEN3, + _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH)); /* * According to the spec the following bits should be @@ -4641,11 +5024,6 @@ static void gen6_init_clock_gating(struct drm_device *dev) g4x_disable_trickle_feed(dev); - /* The default value should be 0x200 according to docs, but the two - * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */ - I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff)); - I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI)); - cpt_init_clock_gating(dev); gen6_check_mch_setup(dev); @@ -4655,14 +5033,17 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) { uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE); + /* + * WaVSThreadDispatchOverride:ivb,vlv + * + * This actually overrides the dispatch + * mode for all thread types. + */ reg &= ~GEN7_FF_SCHED_MASK; reg |= GEN7_FF_TS_SCHED_HW; reg |= GEN7_FF_VS_SCHED_HW; reg |= GEN7_FF_DS_SCHED_HW; - if (IS_HASWELL(dev_priv->dev)) - reg &= ~GEN7_FF_VS_REF_CNT_FFME; - I915_WRITE(GEN7_FF_THREAD_MODE, reg); } @@ -4700,7 +5081,7 @@ static void lpt_suspend_hw(struct drm_device *dev) static void gen8_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - enum pipe i; + enum pipe pipe; I915_WRITE(WM3_LP_ILK, 0); I915_WRITE(WM2_LP_ILK, 0); @@ -4709,8 +5090,19 @@ static void gen8_init_clock_gating(struct drm_device *dev) /* FIXME(BDW): Check all the w/a, some might only apply to * pre-production hw. */ - WARN(!i915_preliminary_hw_support, - "GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n"); + /* WaDisablePartialInstShootdown:bdw */ + I915_WRITE(GEN8_ROW_CHICKEN, + _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); + + /* WaDisableThreadStallDopClockGating:bdw */ + /* FIXME: Unclear whether we really need this on production bdw. */ + I915_WRITE(GEN8_ROW_CHICKEN, + _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); + + /* + * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for + * pre-production hardware + */ I915_WRITE(HALF_SLICE_CHICKEN3, _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS)); I915_WRITE(HALF_SLICE_CHICKEN3, @@ -4726,6 +5118,10 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE)); + /* WaDisableDopClockGating:bdw May not be needed for production */ + I915_WRITE(GEN7_ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); + /* WaSwitchSolVfFArbitrationPriority:bdw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -4734,10 +5130,10 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD); /* WaPsrDPRSUnmaskVBlankInSRD:bdw */ - for_each_pipe(i) { - I915_WRITE(CHICKEN_PIPESL_1(i), - I915_READ(CHICKEN_PIPESL_1(i) | - DPRS_MASK_VBLANK_SRD)); + for_each_pipe(pipe) { + I915_WRITE(CHICKEN_PIPESL_1(pipe), + I915_READ(CHICKEN_PIPESL_1(pipe)) | + BDW_DPRS_MASK_VBLANK_SRD); } /* Use Force Non-Coherent whenever executing a 3D context. This is a @@ -4753,6 +5149,28 @@ static void gen8_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN7_FF_THREAD_MODE, I915_READ(GEN7_FF_THREAD_MODE) & ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); + + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + + I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, + _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); + + /* WaDisableSDEUnitClockGating:bdw */ + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); + + /* Wa4x4STCOptimizationDisable:bdw */ + I915_WRITE(CACHE_MODE_1, + _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE)); } static void haswell_init_clock_gating(struct drm_device *dev) @@ -4761,21 +5179,6 @@ static void haswell_init_clock_gating(struct drm_device *dev) ilk_init_lp_watermarks(dev); - /* According to the spec, bit 13 (RCZUNIT) must be set on IVB. - * This implements the WaDisableRCZUnitClockGating:hsw workaround. - */ - I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE); - - /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); - - /* WaApplyL3ControlAndL3ChickenMode:hsw */ - I915_WRITE(GEN7_L3CNTLREG1, - GEN7_WA_FOR_GEN7_L3_CONTROL); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, - GEN7_WA_L3_CHICKEN_MODE); - /* L3 caching of data atomics doesn't work -- disable it. */ I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); I915_WRITE(HSW_ROW_CHICKEN3, @@ -4787,12 +5190,31 @@ static void haswell_init_clock_gating(struct drm_device *dev) GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); /* WaVSRefCountFullforceMissDisable:hsw */ - gen7_setup_fixed_func_scheduler(dev_priv); + I915_WRITE(GEN7_FF_THREAD_MODE, + I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME); + + /* WaDisable_RenderCache_OperationalFlush:hsw */ + I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); + + /* enable HiZ Raw Stall Optimization */ + I915_WRITE(CACHE_MODE_0_GEN7, + _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); /* WaDisable4x2SubspanOptimization:hsw */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + /* WaSwitchSolVfFArbitrationPriority:hsw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -4825,9 +5247,9 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) if (IS_IVB_GT1(dev)) I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - else - I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2, - _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); + + /* WaDisable_RenderCache_OperationalFlush:ivb */ + I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, @@ -4841,31 +5263,24 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) if (IS_IVB_GT1(dev)) I915_WRITE(GEN7_ROW_CHICKEN2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - else + else { + /* must write both registers */ + I915_WRITE(GEN7_ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); I915_WRITE(GEN7_ROW_CHICKEN2_GT2, _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); - + } /* WaForceL3Serialization:ivb */ I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & ~L3SQ_URB_READ_CAM_MATCH_DISABLE); - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * + /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:ivb workaround. */ I915_WRITE(GEN6_UCGCTL2, - GEN6_RCZUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); + GEN6_RCZUNIT_CLOCK_GATE_DISABLE); /* This is required by WaCatErrorRejectionIssue:ivb */ I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG, @@ -4874,13 +5289,29 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) g4x_disable_trickle_feed(dev); - /* WaVSRefCountFullforceMissDisable:ivb */ gen7_setup_fixed_func_scheduler(dev_priv); + if (0) { /* causes HiZ corruption on ivb:gt1 */ + /* enable HiZ Raw Stall Optimization */ + I915_WRITE(CACHE_MODE_0_GEN7, + _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE)); + } + /* WaDisable4x2SubspanOptimization:ivb */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); + /* + * BSpec recommends 8x4 when MSAA is used, + * however in practice 16x4 seems fastest. + * + * Note that PS/WM thread counts depend on the WIZ hashing + * disable bit, which we don't touch here, but it's good + * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). + */ + I915_WRITE(GEN7_GT_MODE, + GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4); + snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); snpcr &= ~GEN6_MBC_SNPCR_MASK; snpcr |= GEN6_MBC_SNPCR_MED; @@ -4902,13 +5333,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev) mutex_unlock(&dev_priv->rps.hw_lock); switch ((val >> 6) & 3) { case 0: - dev_priv->mem_freq = 800; - break; case 1: - dev_priv->mem_freq = 1066; + dev_priv->mem_freq = 800; break; case 2: - dev_priv->mem_freq = 1333; + dev_priv->mem_freq = 1066; break; case 3: dev_priv->mem_freq = 1333; @@ -4916,6 +5345,10 @@ static void valleyview_init_clock_gating(struct drm_device *dev) } DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq); + dev_priv->vlv_cdclk_freq = valleyview_cur_cdclk(dev_priv); + DRM_DEBUG_DRIVER("Current CD clock rate: %d MHz", + dev_priv->vlv_cdclk_freq); + I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); /* WaDisableEarlyCull:vlv */ @@ -4927,18 +5360,14 @@ static void valleyview_init_clock_gating(struct drm_device *dev) CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE | CHICKEN3_DGMG_DONE_FIX_DISABLE); + /* WaPsdDispatchEnable:vlv */ /* WaDisablePSDDualDispatchEnable:vlv */ I915_WRITE(GEN7_HALF_SLICE_CHICKEN1, _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP | GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE)); - /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */ - I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1, - GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); - - /* WaApplyL3ControlAndL3ChickenMode:vlv */ - I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS); - I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); + /* WaDisable_RenderCache_OperationalFlush:vlv */ + I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); /* WaForceL3Serialization:vlv */ I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) & @@ -4953,51 +5382,95 @@ static void valleyview_init_clock_gating(struct drm_device *dev) I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock - * gating disable must be set. Failure to set it results in - * flickering pixels due to Z write ordering failures after - * some amount of runtime in the Mesa "fire" demo, and Unigine - * Sanctuary and Tropics, and apparently anything else with - * alpha test or pixel discard. - * - * According to the spec, bit 11 (RCCUNIT) must also be set, - * but we didn't debug actual testcases to find it out. - * + gen7_setup_fixed_func_scheduler(dev_priv); + + /* * According to the spec, bit 13 (RCZUNIT) must be set on IVB. * This implements the WaDisableRCZUnitClockGating:vlv workaround. - * - * Also apply WaDisableVDSUnitClockGating:vlv and - * WaDisableRCPBUnitClockGating:vlv. */ I915_WRITE(GEN6_UCGCTL2, - GEN7_VDSUNIT_CLOCK_GATE_DISABLE | - GEN7_TDLUNIT_CLOCK_GATE_DISABLE | - GEN6_RCZUNIT_CLOCK_GATE_DISABLE | - GEN6_RCPBUNIT_CLOCK_GATE_DISABLE | - GEN6_RCCUNIT_CLOCK_GATE_DISABLE); + GEN6_RCZUNIT_CLOCK_GATE_DISABLE); - I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE); + /* WaDisableL3Bank2xClockGate:vlv + * Disabling L3 clock gating- MMIO 940c[25] = 1 + * Set bit 25, to disable L3_BANK_2x_CLK_GATING */ + I915_WRITE(GEN7_UCGCTL4, + I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE); I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); + /* + * BSpec says this must be set, even though + * WaDisable4x2SubspanOptimization isn't listed for VLV. + */ I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE)); /* + * WaIncreaseL3CreditsForVLVB0:vlv + * This is the hardware default actually. + */ + I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); + + /* * WaDisableVLVClockGating_VBIIssue:vlv * Disable clock gating on th GCFG unit to prevent a delay * in the reporting of vblank events. */ - I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff); + I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); +} + +static void cherryview_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); - /* Conservative clock gating settings for now */ - I915_WRITE(0x9400, 0xffffffff); - I915_WRITE(0x9404, 0xffffffff); - I915_WRITE(0x9408, 0xffffffff); - I915_WRITE(0x940c, 0xffffffff); - I915_WRITE(0x9410, 0xffffffff); - I915_WRITE(0x9414, 0xffffffff); - I915_WRITE(0x9418, 0xffffffff); + I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); + + /* WaDisablePartialInstShootdown:chv */ + I915_WRITE(GEN8_ROW_CHICKEN, + _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE)); + + /* WaDisableThreadStallDopClockGating:chv */ + I915_WRITE(GEN8_ROW_CHICKEN, + _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE)); + + /* WaVSRefCountFullforceMissDisable:chv */ + /* WaDSRefCountFullforceMissDisable:chv */ + I915_WRITE(GEN7_FF_THREAD_MODE, + I915_READ(GEN7_FF_THREAD_MODE) & + ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME)); + + /* WaDisableSemaphoreAndSyncFlipWait:chv */ + I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, + _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE)); + + /* WaDisableCSUnitClockGating:chv */ + I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | + GEN6_CSUNIT_CLOCK_GATE_DISABLE); + + /* WaDisableSDEUnitClockGating:chv */ + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); + + /* WaDisableSamplerPowerBypass:chv (pre-production hw) */ + I915_WRITE(HALF_SLICE_CHICKEN3, + _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS)); + + /* WaDisableGunitClockGating:chv (pre-production hw) */ + I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) | + GINT_DIS); + + /* WaDisableFfDopClockGating:chv (pre-production hw) */ + I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL, + _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE)); + + /* WaDisableDopClockGating:chv (pre-production hw) */ + I915_WRITE(GEN7_ROW_CHICKEN2, + _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE)); + I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | + GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE); } static void g4x_init_clock_gating(struct drm_device *dev) @@ -5021,6 +5494,9 @@ static void g4x_init_clock_gating(struct drm_device *dev) I915_WRITE(CACHE_MODE_0, _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE)); + /* WaDisable_RenderCache_OperationalFlush:g4x */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); + g4x_disable_trickle_feed(dev); } @@ -5035,6 +5511,9 @@ static void crestline_init_clock_gating(struct drm_device *dev) I915_WRITE16(DEUC, 0); I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); + + /* WaDisable_RenderCache_OperationalFlush:gen4 */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } static void broadwater_init_clock_gating(struct drm_device *dev) @@ -5049,6 +5528,9 @@ static void broadwater_init_clock_gating(struct drm_device *dev) I915_WRITE(RENCLK_GATE_D2, 0); I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); + + /* WaDisable_RenderCache_OperationalFlush:gen4 */ + I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } static void gen3_init_clock_gating(struct drm_device *dev) @@ -5065,6 +5547,12 @@ static void gen3_init_clock_gating(struct drm_device *dev) /* IIR "flip pending" means done if this bit is set */ I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE)); + + /* interrupts should cause a wake up from C3 */ + I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN)); + + /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ + I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); } static void i85x_init_clock_gating(struct drm_device *dev) @@ -5072,6 +5560,10 @@ static void i85x_init_clock_gating(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); + + /* interrupts should cause a wake up from C3 */ + I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) | + _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE)); } static void i830_init_clock_gating(struct drm_device *dev) @@ -5112,54 +5604,62 @@ void intel_suspend_hw(struct drm_device *dev) * enable it, so check if it's enabled and also check if we've requested it to * be enabled. */ -static bool hsw_power_well_enabled(struct drm_device *dev, +static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_i915_private *dev_priv = dev->dev_private; - return I915_READ(HSW_PWR_WELL_DRIVER) == (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED); } -bool intel_display_power_enabled_sw(struct drm_device *dev, - enum intel_display_power_domain domain) +bool intel_display_power_enabled_unlocked(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_power_domains *power_domains; - - power_domains = &dev_priv->power_domains; - - return power_domains->domain_use_count[domain]; -} - -bool intel_display_power_enabled(struct drm_device *dev, - enum intel_display_power_domain domain) -{ - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; struct i915_power_well *power_well; bool is_enabled; int i; + if (dev_priv->pm.suspended) + return false; + power_domains = &dev_priv->power_domains; is_enabled = true; - mutex_lock(&power_domains->lock); for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { if (power_well->always_on) continue; - if (!power_well->is_enabled(dev, power_well)) { + if (!power_well->hw_enabled) { is_enabled = false; break; } } - mutex_unlock(&power_domains->lock); return is_enabled; } +bool intel_display_power_enabled(struct drm_i915_private *dev_priv, + enum intel_display_power_domain domain) +{ + struct i915_power_domains *power_domains; + bool ret; + + power_domains = &dev_priv->power_domains; + + mutex_lock(&power_domains->lock); + ret = intel_display_power_enabled_unlocked(dev_priv, domain); + mutex_unlock(&power_domains->lock); + + return ret; +} + +/* + * Starting with Haswell, we have a "Power Down Well" that can be turned off + * when not needed anymore. We have 4 registers that can request the power well + * to be enabled, and it will only be disabled if none of the registers is + * requesting it to be enabled. + */ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; @@ -5196,35 +5696,12 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) } } -static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv) -{ - struct drm_device *dev = dev_priv->dev; - enum pipe p; - unsigned long irqflags; - - /* - * After this, the registers on the pipes that are part of the power - * well will become zero, so we have to adjust our counters according to - * that. - * - * FIXME: Should we do this in general in drm_vblank_post_modeset? - */ - spin_lock_irqsave(&dev->vbl_lock, irqflags); - for_each_pipe(p) - if (p != PIPE_A) - dev->vblank[p].last = 0; - spin_unlock_irqrestore(&dev->vbl_lock, irqflags); -} - -static void hsw_set_power_well(struct drm_device *dev, +static void hsw_set_power_well(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; bool is_enabled, enable_requested; uint32_t tmp; - WARN_ON(dev_priv->pc8.enabled); - tmp = I915_READ(HSW_PWR_WELL_DRIVER); is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED; enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST; @@ -5247,61 +5724,268 @@ static void hsw_set_power_well(struct drm_device *dev, I915_WRITE(HSW_PWR_WELL_DRIVER, 0); POSTING_READ(HSW_PWR_WELL_DRIVER); DRM_DEBUG_KMS("Requesting to disable the power well\n"); - - hsw_power_well_post_disable(dev_priv); } } } -static void __intel_power_well_get(struct drm_device *dev, +static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_i915_private *dev_priv = dev->dev_private; + hsw_set_power_well(dev_priv, power_well, power_well->count > 0); + + /* + * We're taking over the BIOS, so clear any requests made by it since + * the driver is in charge now. + */ + if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) + I915_WRITE(HSW_PWR_WELL_BIOS, 0); +} + +static void hsw_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + hsw_set_power_well(dev_priv, power_well, true); +} + +static void hsw_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + hsw_set_power_well(dev_priv, power_well, false); +} + +static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ +} + +static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + return true; +} + +void __vlv_set_power_well(struct drm_i915_private *dev_priv, + enum punit_power_well power_well_id, bool enable) +{ + struct drm_device *dev = dev_priv->dev; + u32 mask; + u32 state; + u32 ctrl; + enum pipe pipe; - if (!power_well->count++ && power_well->set) { - hsw_disable_package_c8(dev_priv); - power_well->set(dev, power_well, true); + if (power_well_id == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (enable) { + /* + * Enable the CRI clock source so we can get at the + * display and the reference clock for VGA + * hotplug / manual detection. + */ + I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | + DPLL_REFA_CLK_ENABLE_VLV | + DPLL_INTEGRATED_CRI_CLK_VLV); + udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ + } else { + for_each_pipe(pipe) + assert_pll_disabled(dev_priv, pipe); + /* Assert common reset */ + I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & + ~DPIO_CMNRST); + } } + + mask = PUNIT_PWRGT_MASK(power_well_id); + state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) : + PUNIT_PWRGT_PWR_GATE(power_well_id); + + mutex_lock(&dev_priv->rps.hw_lock); + +#define COND \ + ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state) + + if (COND) + goto out; + + ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL); + ctrl &= ~mask; + ctrl |= state; + vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl); + + if (wait_for(COND, 100)) + DRM_ERROR("timout setting power well state %08x (%08x)\n", + state, + vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL)); + +#undef COND + +out: + mutex_unlock(&dev_priv->rps.hw_lock); + + /* + * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx - + * 6. De-assert cmn_reset/side_reset. Same as VLV X0. + * a. GUnit 0x2110 bit[0] set to 1 (def 0) + * b. The other bits such as sfr settings / modesel may all + * be set to 0. + * + * This should only be done on init and resume from S3 with + * both PLLs disabled, or we risk losing DPIO and PLL + * synchronization. + */ + if (power_well_id == PUNIT_POWER_WELL_DPIO_CMN_BC && enable) + I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST); +} + +static void vlv_set_power_well(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well, bool enable) +{ + enum punit_power_well power_well_id = power_well->data; + + __vlv_set_power_well(dev_priv, power_well_id, enable); } -static void __intel_power_well_put(struct drm_device *dev, +static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_i915_private *dev_priv = dev->dev_private; + vlv_set_power_well(dev_priv, power_well, power_well->count > 0); +} + +static void vlv_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + vlv_set_power_well(dev_priv, power_well, true); +} + +static void vlv_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + vlv_set_power_well(dev_priv, power_well, false); +} + +static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + int power_well_id = power_well->data; + bool enabled = false; + u32 mask; + u32 state; + u32 ctrl; + + mask = PUNIT_PWRGT_MASK(power_well_id); + ctrl = PUNIT_PWRGT_PWR_ON(power_well_id); - WARN_ON(!power_well->count); + mutex_lock(&dev_priv->rps.hw_lock); - if (!--power_well->count && power_well->set && - i915_disable_power_well) { - power_well->set(dev, power_well, false); - hsw_enable_package_c8(dev_priv); + state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask; + /* + * We only ever set the power-on and power-gate states, anything + * else is unexpected. + */ + WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) && + state != PUNIT_PWRGT_PWR_GATE(power_well_id)); + if (state == ctrl) + enabled = true; + + /* + * A transient state at this point would mean some unexpected party + * is poking at the power controls too. + */ + ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask; + WARN_ON(ctrl != state); + + mutex_unlock(&dev_priv->rps.hw_lock); + + return enabled; +} + +static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + + vlv_set_power_well(dev_priv, power_well, true); + + spin_lock_irq(&dev_priv->irq_lock); + valleyview_enable_display_irqs(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); + + /* + * During driver initialization/resume we can avoid restoring the + * part of the HW/SW state that will be inited anyway explicitly. + */ + if (dev_priv->power_domains.initializing) + return; + + intel_hpd_init(dev_priv->dev); + + i915_redisable_vga_power_on(dev_priv->dev); +} + +static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + + spin_lock_irq(&dev_priv->irq_lock); + valleyview_disable_display_irqs(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); + + vlv_set_power_well(dev_priv, power_well, false); +} + +static void check_power_well_state(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + bool enabled = power_well->ops->is_enabled(dev_priv, power_well); + + if (power_well->always_on || !i915.disable_power_well) { + if (!enabled) + goto mismatch; + + return; } + + if (enabled != (power_well->count > 0)) + goto mismatch; + + return; + +mismatch: + WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n", + power_well->name, power_well->always_on, enabled, + power_well->count, i915.disable_power_well); } -void intel_display_power_get(struct drm_device *dev, +void intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; struct i915_power_well *power_well; int i; + intel_runtime_pm_get(dev_priv); + power_domains = &dev_priv->power_domains; mutex_lock(&power_domains->lock); - for_each_power_well(i, power_well, BIT(domain), power_domains) - __intel_power_well_get(dev, power_well); + for_each_power_well(i, power_well, BIT(domain), power_domains) { + if (!power_well->count++) { + DRM_DEBUG_KMS("enabling %s\n", power_well->name); + power_well->ops->enable(dev_priv, power_well); + power_well->hw_enabled = true; + } + + check_power_well_state(dev_priv, power_well); + } power_domains->domain_use_count[domain]++; mutex_unlock(&power_domains->lock); } -void intel_display_power_put(struct drm_device *dev, +void intel_display_power_put(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains; struct i915_power_well *power_well; int i; @@ -5313,61 +5997,165 @@ void intel_display_power_put(struct drm_device *dev, WARN_ON(!power_domains->domain_use_count[domain]); power_domains->domain_use_count[domain]--; - for_each_power_well_rev(i, power_well, BIT(domain), power_domains) - __intel_power_well_put(dev, power_well); + for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { + WARN_ON(!power_well->count); + + if (!--power_well->count && i915.disable_power_well) { + DRM_DEBUG_KMS("disabling %s\n", power_well->name); + power_well->hw_enabled = false; + power_well->ops->disable(dev_priv, power_well); + } + + check_power_well_state(dev_priv, power_well); + } mutex_unlock(&power_domains->lock); + + intel_runtime_pm_put(dev_priv); } static struct i915_power_domains *hsw_pwr; /* Display audio driver power well request */ -void i915_request_power_well(void) +int i915_request_power_well(void) { struct drm_i915_private *dev_priv; - if (WARN_ON(!hsw_pwr)) - return; + if (!hsw_pwr) + return -ENODEV; dev_priv = container_of(hsw_pwr, struct drm_i915_private, power_domains); - intel_display_power_get(dev_priv->dev, POWER_DOMAIN_AUDIO); + intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); + return 0; } EXPORT_SYMBOL_GPL(i915_request_power_well); /* Display audio driver power well release */ -void i915_release_power_well(void) +int i915_release_power_well(void) { struct drm_i915_private *dev_priv; - if (WARN_ON(!hsw_pwr)) - return; + if (!hsw_pwr) + return -ENODEV; dev_priv = container_of(hsw_pwr, struct drm_i915_private, power_domains); - intel_display_power_put(dev_priv->dev, POWER_DOMAIN_AUDIO); + intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); + return 0; } EXPORT_SYMBOL_GPL(i915_release_power_well); +/* + * Private interface for the audio driver to get CDCLK in kHz. + * + * Caller must request power well using i915_request_power_well() prior to + * making the call. + */ +int i915_get_cdclk_freq(void) +{ + struct drm_i915_private *dev_priv; + + if (!hsw_pwr) + return -ENODEV; + + dev_priv = container_of(hsw_pwr, struct drm_i915_private, + power_domains); + + return intel_ddi_get_cdclk_freq(dev_priv); +} +EXPORT_SYMBOL_GPL(i915_get_cdclk_freq); + + +#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1) + +#define HSW_ALWAYS_ON_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PIPE_A) | \ + BIT(POWER_DOMAIN_TRANSCODER_EDP) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_CRT) | \ + BIT(POWER_DOMAIN_INIT)) +#define HSW_DISPLAY_POWER_DOMAINS ( \ + (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) | \ + BIT(POWER_DOMAIN_INIT)) + +#define BDW_ALWAYS_ON_POWER_DOMAINS ( \ + HSW_ALWAYS_ON_POWER_DOMAINS | \ + BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER)) +#define BDW_DISPLAY_POWER_DOMAINS ( \ + (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT) +#define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK + +#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_PORT_CRT) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) | \ + BIT(POWER_DOMAIN_INIT)) + +static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { + .sync_hw = i9xx_always_on_power_well_noop, + .enable = i9xx_always_on_power_well_noop, + .disable = i9xx_always_on_power_well_noop, + .is_enabled = i9xx_always_on_power_well_enabled, +}; + static struct i915_power_well i9xx_always_on_power_well[] = { { .name = "always-on", .always_on = 1, .domains = POWER_DOMAIN_MASK, + .ops = &i9xx_always_on_power_well_ops, }, }; +static const struct i915_power_well_ops hsw_power_well_ops = { + .sync_hw = hsw_power_well_sync_hw, + .enable = hsw_power_well_enable, + .disable = hsw_power_well_disable, + .is_enabled = hsw_power_well_enabled, +}; + static struct i915_power_well hsw_power_wells[] = { { .name = "always-on", .always_on = 1, .domains = HSW_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, }, { .name = "display", - .domains = POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS, - .is_enabled = hsw_power_well_enabled, - .set = hsw_set_power_well, + .domains = HSW_DISPLAY_POWER_DOMAINS, + .ops = &hsw_power_well_ops, }, }; @@ -5376,12 +6164,83 @@ static struct i915_power_well bdw_power_wells[] = { .name = "always-on", .always_on = 1, .domains = BDW_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, }, { .name = "display", - .domains = POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS, - .is_enabled = hsw_power_well_enabled, - .set = hsw_set_power_well, + .domains = BDW_DISPLAY_POWER_DOMAINS, + .ops = &hsw_power_well_ops, + }, +}; + +static const struct i915_power_well_ops vlv_display_power_well_ops = { + .sync_hw = vlv_power_well_sync_hw, + .enable = vlv_display_power_well_enable, + .disable = vlv_display_power_well_disable, + .is_enabled = vlv_power_well_enabled, +}; + +static const struct i915_power_well_ops vlv_dpio_power_well_ops = { + .sync_hw = vlv_power_well_sync_hw, + .enable = vlv_power_well_enable, + .disable = vlv_power_well_disable, + .is_enabled = vlv_power_well_enabled, +}; + +static struct i915_power_well vlv_power_wells[] = { + { + .name = "always-on", + .always_on = 1, + .domains = VLV_ALWAYS_ON_POWER_DOMAINS, + .ops = &i9xx_always_on_power_well_ops, + }, + { + .name = "display", + .domains = VLV_DISPLAY_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DISP2D, + .ops = &vlv_display_power_well_ops, + }, + { + .name = "dpio-tx-b-01", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, + }, + { + .name = "dpio-tx-b-23", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, + }, + { + .name = "dpio-tx-c-01", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, + }, + { + .name = "dpio-tx-c-23", + .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | + VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, + .ops = &vlv_dpio_power_well_ops, + .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, + }, + { + .name = "dpio-common", + .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS, + .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .ops = &vlv_dpio_power_well_ops, }, }; @@ -5390,9 +6249,8 @@ static struct i915_power_well bdw_power_wells[] = { (power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \ }) -int intel_power_domains_init(struct drm_device *dev) +int intel_power_domains_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; mutex_init(&power_domains->lock); @@ -5401,12 +6259,14 @@ int intel_power_domains_init(struct drm_device *dev) * The enabling order will be from lower to higher indexed wells, * the disabling order is reversed. */ - if (IS_HASWELL(dev)) { + if (IS_HASWELL(dev_priv->dev)) { set_power_wells(power_domains, hsw_power_wells); hsw_pwr = power_domains; - } else if (IS_BROADWELL(dev)) { + } else if (IS_BROADWELL(dev_priv->dev)) { set_power_wells(power_domains, bdw_power_wells); hsw_pwr = power_domains; + } else if (IS_VALLEYVIEW(dev_priv->dev)) { + set_power_wells(power_domains, vlv_power_wells); } else { set_power_wells(power_domains, i9xx_always_on_power_well); } @@ -5414,58 +6274,45 @@ int intel_power_domains_init(struct drm_device *dev) return 0; } -void intel_power_domains_remove(struct drm_device *dev) +void intel_power_domains_remove(struct drm_i915_private *dev_priv) { hsw_pwr = NULL; } -static void intel_power_domains_resume(struct drm_device *dev) +static void intel_power_domains_resume(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct i915_power_domains *power_domains = &dev_priv->power_domains; struct i915_power_well *power_well; int i; mutex_lock(&power_domains->lock); for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) { - if (power_well->set) - power_well->set(dev, power_well, power_well->count > 0); + power_well->ops->sync_hw(dev_priv, power_well); + power_well->hw_enabled = power_well->ops->is_enabled(dev_priv, + power_well); } mutex_unlock(&power_domains->lock); } -/* - * Starting with Haswell, we have a "Power Down Well" that can be turned off - * when not needed anymore. We have 4 registers that can request the power well - * to be enabled, and it will only be disabled if none of the registers is - * requesting it to be enabled. - */ -void intel_power_domains_init_hw(struct drm_device *dev) +void intel_power_domains_init_hw(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct i915_power_domains *power_domains = &dev_priv->power_domains; + power_domains->initializing = true; /* For now, we need the power well to be always enabled. */ - intel_display_set_init_power(dev, true); - intel_power_domains_resume(dev); - - if (!(IS_HASWELL(dev) || IS_BROADWELL(dev))) - return; - - /* We're taking over the BIOS, so clear any requests made by it since - * the driver is in charge now. */ - if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) - I915_WRITE(HSW_PWR_WELL_BIOS, 0); + intel_display_set_init_power(dev_priv, true); + intel_power_domains_resume(dev_priv); + power_domains->initializing = false; } -/* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */ void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv) { - hsw_disable_package_c8(dev_priv); + intel_runtime_pm_get(dev_priv); } void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv) { - hsw_enable_package_c8(dev_priv); + intel_runtime_pm_put(dev_priv); } void intel_runtime_pm_get(struct drm_i915_private *dev_priv) @@ -5480,6 +6327,18 @@ void intel_runtime_pm_get(struct drm_i915_private *dev_priv) WARN(dev_priv->pm.suspended, "Device still suspended.\n"); } +void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = dev_priv->dev; + struct device *device = &dev->pdev->dev; + + if (!HAS_RUNTIME_PM(dev)) + return; + + WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n"); + pm_runtime_get_noresume(device); +} + void intel_runtime_pm_put(struct drm_i915_private *dev_priv) { struct drm_device *dev = dev_priv->dev; @@ -5497,16 +6356,25 @@ void intel_init_runtime_pm(struct drm_i915_private *dev_priv) struct drm_device *dev = dev_priv->dev; struct device *device = &dev->pdev->dev; - dev_priv->pm.suspended = false; - if (!HAS_RUNTIME_PM(dev)) return; pm_runtime_set_active(device); + /* + * RPM depends on RC6 to save restore the GT HW context, so make RC6 a + * requirement. + */ + if (!intel_enable_rc6(dev)) { + DRM_INFO("RC6 disabled, disabling runtime PM support\n"); + return; + } + pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */ pm_runtime_mark_last_busy(device); pm_runtime_use_autosuspend(device); + + pm_runtime_put_autosuspend(device); } void intel_fini_runtime_pm(struct drm_i915_private *dev_priv) @@ -5517,6 +6385,9 @@ void intel_fini_runtime_pm(struct drm_i915_private *dev_priv) if (!HAS_RUNTIME_PM(dev)) return; + if (!intel_enable_rc6(dev)) + return; + /* Make sure we're not suspended first. */ pm_runtime_get_sync(device); pm_runtime_disable(device); @@ -5558,7 +6429,7 @@ void intel_init_pm(struct drm_device *dev) /* For FIFO watermark updates */ if (HAS_PCH_SPLIT(dev)) { - intel_setup_wm_latency(dev); + ilk_setup_wm_latency(dev); if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] && dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || @@ -5581,6 +6452,10 @@ void intel_init_pm(struct drm_device *dev) dev_priv->display.init_clock_gating = haswell_init_clock_gating; else if (INTEL_INFO(dev)->gen == 8) dev_priv->display.init_clock_gating = gen8_init_clock_gating; + } else if (IS_CHERRYVIEW(dev)) { + dev_priv->display.update_wm = valleyview_update_wm; + dev_priv->display.init_clock_gating = + cherryview_init_clock_gating; } else if (IS_VALLEYVIEW(dev)) { dev_priv->display.update_wm = valleyview_update_wm; dev_priv->display.init_clock_gating = @@ -5729,13 +6604,9 @@ void intel_pm_setup(struct drm_device *dev) mutex_init(&dev_priv->rps.hw_lock); - mutex_init(&dev_priv->pc8.lock); - dev_priv->pc8.requirements_met = false; - dev_priv->pc8.gpu_idle = false; - dev_priv->pc8.irqs_disabled = false; - dev_priv->pc8.enabled = false; - dev_priv->pc8.disable_count = 2; /* requirements_met + gpu_idle */ - INIT_DELAYED_WORK(&dev_priv->pc8.enable_work, hsw_enable_pc8_work); INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work, intel_gen6_powersave_work); + + dev_priv->pm.suspended = false; + dev_priv->pm.irqs_disabled = false; } diff --git a/drivers/gpu/drm/i915/intel_renderstate.h b/drivers/gpu/drm/i915/intel_renderstate.h new file mode 100644 index 00000000000..a5e783a9928 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_renderstate.h @@ -0,0 +1,48 @@ +/* + * Copyright © 2014 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _INTEL_RENDERSTATE_H +#define _INTEL_RENDERSTATE_H + +#include <linux/types.h> + +struct intel_renderstate_rodata { + const u32 *reloc; + const u32 reloc_items; + const u32 *batch; + const u32 batch_items; +}; + +extern const struct intel_renderstate_rodata gen6_null_state; +extern const struct intel_renderstate_rodata gen7_null_state; +extern const struct intel_renderstate_rodata gen8_null_state; + +#define RO_RENDERSTATE(_g) \ + const struct intel_renderstate_rodata gen ## _g ## _null_state = { \ + .reloc = gen ## _g ## _null_state_relocs, \ + .reloc_items = sizeof(gen ## _g ## _null_state_relocs)/4, \ + .batch = gen ## _g ## _null_state_batch, \ + .batch_items = sizeof(gen ## _g ## _null_state_batch)/4, \ + } + +#endif /* INTEL_RENDERSTATE_H */ diff --git a/drivers/gpu/drm/i915/intel_renderstate_gen6.c b/drivers/gpu/drm/i915/intel_renderstate_gen6.c new file mode 100644 index 00000000000..740538ad097 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_renderstate_gen6.c @@ -0,0 +1,289 @@ +#include "intel_renderstate.h" + +static const u32 gen6_null_state_relocs[] = { + 0x00000020, + 0x00000024, + 0x0000002c, + 0x000001e0, + 0x000001e4, +}; + +static const u32 gen6_null_state_batch[] = { + 0x69040000, + 0x790d0001, + 0x00000000, + 0x00000000, + 0x78180000, + 0x00000001, + 0x61010008, + 0x00000000, + 0x00000001, /* reloc */ + 0x00000001, /* reloc */ + 0x00000000, + 0x00000001, /* reloc */ + 0x00000000, + 0x00000001, + 0x00000000, + 0x00000001, + 0x61020000, + 0x00000000, + 0x78050001, + 0x00000018, + 0x00000000, + 0x780d1002, + 0x00000000, + 0x00000000, + 0x00000420, + 0x78150003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78100004, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78160003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78110005, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78120002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78170003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x79050005, + 0xe0040000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x79100000, + 0x00000000, + 0x79000002, + 0xffffffff, + 0x00000000, + 0x00000000, + 0x780e0002, + 0x00000441, + 0x00000401, + 0x00000401, + 0x78021002, + 0x00000000, + 0x00000000, + 0x00000400, + 0x78130012, + 0x00400810, + 0x00000000, + 0x20000000, + 0x04000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78140007, + 0x00000280, + 0x08080000, + 0x00000000, + 0x00060000, + 0x4e080002, + 0x00100400, + 0x00000000, + 0x00000000, + 0x78090005, + 0x02000000, + 0x22220000, + 0x02f60000, + 0x11330000, + 0x02850004, + 0x11220000, + 0x78011002, + 0x00000000, + 0x00000000, + 0x00000200, + 0x78080003, + 0x00002000, + 0x00000448, /* reloc */ + 0x00000448, /* reloc */ + 0x00000000, + 0x05000000, /* cmds end */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000220, /* state start */ + 0x00000240, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0060005a, + 0x204077be, + 0x000000c0, + 0x008d0040, + 0x0060005a, + 0x206077be, + 0x000000c0, + 0x008d0080, + 0x0060005a, + 0x208077be, + 0x000000d0, + 0x008d0040, + 0x0060005a, + 0x20a077be, + 0x000000d0, + 0x008d0080, + 0x00000201, + 0x20080061, + 0x00000000, + 0x00000000, + 0x00600001, + 0x20200022, + 0x008d0000, + 0x00000000, + 0x02800031, + 0x21c01cc9, + 0x00000020, + 0x0a8a0001, + 0x00600001, + 0x204003be, + 0x008d01c0, + 0x00000000, + 0x00600001, + 0x206003be, + 0x008d01e0, + 0x00000000, + 0x00600001, + 0x208003be, + 0x008d0200, + 0x00000000, + 0x00600001, + 0x20a003be, + 0x008d0220, + 0x00000000, + 0x00600001, + 0x20c003be, + 0x008d0240, + 0x00000000, + 0x00600001, + 0x20e003be, + 0x008d0260, + 0x00000000, + 0x00600001, + 0x210003be, + 0x008d0280, + 0x00000000, + 0x00600001, + 0x212003be, + 0x008d02a0, + 0x00000000, + 0x05800031, + 0x24001cc8, + 0x00000040, + 0x90019000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0000007e, + 0x00000000, + 0x00000000, + 0x00000000, + 0x30000000, + 0x00000124, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0xf99a130c, + 0x799a130c, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x80000031, + 0x00000003, + 0x00000000, /* state end */ +}; + +RO_RENDERSTATE(6); diff --git a/drivers/gpu/drm/i915/intel_renderstate_gen7.c b/drivers/gpu/drm/i915/intel_renderstate_gen7.c new file mode 100644 index 00000000000..6fa7ff2a129 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_renderstate_gen7.c @@ -0,0 +1,253 @@ +#include "intel_renderstate.h" + +static const u32 gen7_null_state_relocs[] = { + 0x0000000c, + 0x00000010, + 0x00000018, + 0x000001ec, +}; + +static const u32 gen7_null_state_batch[] = { + 0x69040000, + 0x61010008, + 0x00000000, + 0x00000001, /* reloc */ + 0x00000001, /* reloc */ + 0x00000000, + 0x00000001, /* reloc */ + 0x00000000, + 0x00000001, + 0x00000000, + 0x00000001, + 0x790d0002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78180000, + 0x00000001, + 0x79160000, + 0x00000008, + 0x78300000, + 0x02010040, + 0x78310000, + 0x04000000, + 0x78320000, + 0x04000000, + 0x78330000, + 0x02000000, + 0x78100004, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781b0005, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781c0002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781d0004, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78110005, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78120002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78210000, + 0x00000000, + 0x78130005, + 0x00000000, + 0x20000000, + 0x04000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78140001, + 0x20000800, + 0x00000000, + 0x781e0001, + 0x00000000, + 0x00000000, + 0x78050005, + 0xe0040000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78040001, + 0x00000000, + 0x00000000, + 0x78240000, + 0x00000240, + 0x78230000, + 0x00000260, + 0x782f0000, + 0x00000280, + 0x781f000c, + 0x00400810, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78200006, + 0x000002c0, + 0x08080000, + 0x00000000, + 0x28000402, + 0x00060000, + 0x00000000, + 0x00000000, + 0x78090005, + 0x02000000, + 0x22220000, + 0x02f60000, + 0x11230000, + 0x02f60004, + 0x11230000, + 0x78080003, + 0x00006008, + 0x00000340, /* reloc */ + 0xffffffff, + 0x00000000, + 0x782a0000, + 0x00000360, + 0x79000002, + 0xffffffff, + 0x00000000, + 0x00000000, + 0x7b000005, + 0x0000000f, + 0x00000003, + 0x00000000, + 0x00000001, + 0x00000000, + 0x00000000, + 0x05000000, /* cmds end */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000031, /* state start */ + 0x00000003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0xf99a130c, + 0x799a130c, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000492, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0080005a, + 0x2e2077bd, + 0x000000c0, + 0x008d0040, + 0x0080005a, + 0x2e6077bd, + 0x000000d0, + 0x008d0040, + 0x02800031, + 0x21801fa9, + 0x008d0e20, + 0x08840001, + 0x00800001, + 0x2e2003bd, + 0x008d0180, + 0x00000000, + 0x00800001, + 0x2e6003bd, + 0x008d01c0, + 0x00000000, + 0x00800001, + 0x2ea003bd, + 0x008d0200, + 0x00000000, + 0x00800001, + 0x2ee003bd, + 0x008d0240, + 0x00000000, + 0x05800031, + 0x20001fa8, + 0x008d0e20, + 0x90031000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000380, + 0x000003a0, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, /* state end */ +}; + +RO_RENDERSTATE(7); diff --git a/drivers/gpu/drm/i915/intel_renderstate_gen8.c b/drivers/gpu/drm/i915/intel_renderstate_gen8.c new file mode 100644 index 00000000000..5c875615d42 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_renderstate_gen8.c @@ -0,0 +1,479 @@ +#include "intel_renderstate.h" + +static const u32 gen8_null_state_relocs[] = { + 0x00000048, + 0x00000050, + 0x00000060, + 0x000003ec, +}; + +static const u32 gen8_null_state_batch[] = { + 0x69040000, + 0x61020001, + 0x00000000, + 0x00000000, + 0x79120000, + 0x00000000, + 0x79130000, + 0x00000000, + 0x79140000, + 0x00000000, + 0x79150000, + 0x00000000, + 0x79160000, + 0x00000000, + 0x6101000e, + 0x00000001, + 0x00000000, + 0x00000001, + 0x00000001, /* reloc */ + 0x00000000, + 0x00000001, /* reloc */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000001, /* reloc */ + 0x00000000, + 0xfffff001, + 0x00001001, + 0xfffff001, + 0x00001001, + 0x78230000, + 0x000006e0, + 0x78210000, + 0x00000700, + 0x78300000, + 0x08010040, + 0x78330000, + 0x08000000, + 0x78310000, + 0x08000000, + 0x78320000, + 0x08000000, + 0x78240000, + 0x00000641, + 0x780e0000, + 0x00000601, + 0x780d0000, + 0x00000000, + 0x78180000, + 0x00000001, + 0x78520003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78190009, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781b0007, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78270000, + 0x00000000, + 0x782c0000, + 0x00000000, + 0x781c0002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78160009, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78110008, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78290000, + 0x00000000, + 0x782e0000, + 0x00000000, + 0x781a0009, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781d0007, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78280000, + 0x00000000, + 0x782d0000, + 0x00000000, + 0x78260000, + 0x00000000, + 0x782b0000, + 0x00000000, + 0x78150009, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78100007, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781e0003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78120002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x781f0002, + 0x30400820, + 0x00000000, + 0x00000000, + 0x78510009, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78500003, + 0x00210000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78130002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x782a0000, + 0x00000480, + 0x782f0000, + 0x00000540, + 0x78140000, + 0x00000800, + 0x78170009, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x7820000a, + 0x00000580, + 0x00000000, + 0x08080000, + 0x00000000, + 0x00000000, + 0x1f000002, + 0x00060000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x784d0000, + 0x40000000, + 0x784f0000, + 0x80000100, + 0x780f0000, + 0x00000740, + 0x78050006, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78070003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78060003, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x78040001, + 0x00000000, + 0x00000001, + 0x79000002, + 0xffffffff, + 0x00000000, + 0x00000000, + 0x78080003, + 0x00006000, + 0x000005e0, /* reloc */ + 0x00000000, + 0x00000000, + 0x78090005, + 0x02000000, + 0x22220000, + 0x02f60000, + 0x11230000, + 0x02850004, + 0x11230000, + 0x784b0000, + 0x0000000f, + 0x78490001, + 0x00000000, + 0x00000000, + 0x7b000005, + 0x00000000, + 0x00000003, + 0x00000000, + 0x00000001, + 0x00000000, + 0x00000000, + 0x05000000, /* cmds end */ + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x000004c0, /* state start */ + 0x00000500, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000092, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x0060005a, + 0x21403ae8, + 0x3a0000c0, + 0x008d0040, + 0x0060005a, + 0x21603ae8, + 0x3a0000c0, + 0x008d0080, + 0x0060005a, + 0x21803ae8, + 0x3a0000d0, + 0x008d0040, + 0x0060005a, + 0x21a03ae8, + 0x3a0000d0, + 0x008d0080, + 0x02800031, + 0x2e0022e8, + 0x0e000140, + 0x08840001, + 0x05800031, + 0x200022e0, + 0x0e000e00, + 0x90031000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x06200000, + 0x00000002, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0xf99a130c, + 0x799a130c, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x3f800000, + 0x00000000, + 0x3f800000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, /* state end */ +}; + +RO_RENDERSTATE(8); diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index b7f1742caf8..279488addf3 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -33,26 +33,44 @@ #include "i915_trace.h" #include "intel_drv.h" -static inline int ring_space(struct intel_ring_buffer *ring) +/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill, + * but keeps the logic simple. Indeed, the whole purpose of this macro is just + * to give some inclination as to some of the magic values used in the various + * workarounds! + */ +#define CACHELINE_BYTES 64 + +static inline int __ring_space(int head, int tail, int size) { - int space = (ring->head & HEAD_ADDR) - (ring->tail + I915_RING_FREE_SPACE); + int space = head - (tail + I915_RING_FREE_SPACE); if (space < 0) - space += ring->size; + space += size; return space; } -void __intel_ring_advance(struct intel_ring_buffer *ring) +static inline int ring_space(struct intel_engine_cs *ring) +{ + struct intel_ringbuffer *ringbuf = ring->buffer; + return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size); +} + +static bool intel_ring_stopped(struct intel_engine_cs *ring) { struct drm_i915_private *dev_priv = ring->dev->dev_private; + return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring); +} - ring->tail &= ring->size - 1; - if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring)) +void __intel_ring_advance(struct intel_engine_cs *ring) +{ + struct intel_ringbuffer *ringbuf = ring->buffer; + ringbuf->tail &= ringbuf->size - 1; + if (intel_ring_stopped(ring)) return; - ring->write_tail(ring, ring->tail); + ring->write_tail(ring, ringbuf->tail); } static int -gen2_render_ring_flush(struct intel_ring_buffer *ring, +gen2_render_ring_flush(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains) { @@ -78,7 +96,7 @@ gen2_render_ring_flush(struct intel_ring_buffer *ring, } static int -gen4_render_ring_flush(struct intel_ring_buffer *ring, +gen4_render_ring_flush(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains) { @@ -173,9 +191,9 @@ gen4_render_ring_flush(struct intel_ring_buffer *ring, * really our business. That leaves only stall at scoreboard. */ static int -intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring) +intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring) { - u32 scratch_addr = ring->scratch.gtt_offset + 128; + u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; @@ -208,11 +226,11 @@ intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring) } static int -gen6_render_ring_flush(struct intel_ring_buffer *ring, +gen6_render_ring_flush(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains) { u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 128; + u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; /* Force SNB workarounds for PIPE_CONTROL flushes */ @@ -260,7 +278,7 @@ gen6_render_ring_flush(struct intel_ring_buffer *ring, } static int -gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring) +gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring) { int ret; @@ -278,7 +296,7 @@ gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring) return 0; } -static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value) +static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value) { int ret; @@ -302,11 +320,11 @@ static int gen7_ring_fbc_flush(struct intel_ring_buffer *ring, u32 value) } static int -gen7_render_ring_flush(struct intel_ring_buffer *ring, +gen7_render_ring_flush(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains) { u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 128; + u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; /* @@ -363,11 +381,11 @@ gen7_render_ring_flush(struct intel_ring_buffer *ring, } static int -gen8_render_ring_flush(struct intel_ring_buffer *ring, +gen8_render_ring_flush(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains) { u32 flags = 0; - u32 scratch_addr = ring->scratch.gtt_offset + 128; + u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; flags |= PIPE_CONTROL_CS_STALL; @@ -403,23 +421,30 @@ gen8_render_ring_flush(struct intel_ring_buffer *ring, } -static void ring_write_tail(struct intel_ring_buffer *ring, +static void ring_write_tail(struct intel_engine_cs *ring, u32 value) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = ring->dev->dev_private; I915_WRITE_TAIL(ring, value); } -u32 intel_ring_get_active_head(struct intel_ring_buffer *ring) +u64 intel_ring_get_active_head(struct intel_engine_cs *ring) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; - u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ? - RING_ACTHD(ring->mmio_base) : ACTHD; + struct drm_i915_private *dev_priv = ring->dev->dev_private; + u64 acthd; - return I915_READ(acthd_reg); + if (INTEL_INFO(ring->dev)->gen >= 8) + acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base), + RING_ACTHD_UDW(ring->mmio_base)); + else if (INTEL_INFO(ring->dev)->gen >= 4) + acthd = I915_READ(RING_ACTHD(ring->mmio_base)); + else + acthd = I915_READ(ACTHD); + + return acthd; } -static void ring_setup_phys_status_page(struct intel_ring_buffer *ring) +static void ring_setup_phys_status_page(struct intel_engine_cs *ring) { struct drm_i915_private *dev_priv = ring->dev->dev_private; u32 addr; @@ -430,30 +455,42 @@ static void ring_setup_phys_status_page(struct intel_ring_buffer *ring) I915_WRITE(HWS_PGA, addr); } -static int init_ring_common(struct intel_ring_buffer *ring) +static bool stop_ring(struct intel_engine_cs *ring) { - struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj = ring->obj; - int ret = 0; - u32 head; + struct drm_i915_private *dev_priv = to_i915(ring->dev); - gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); - - if (I915_NEED_GFX_HWS(dev)) - intel_ring_setup_status_page(ring); - else - ring_setup_phys_status_page(ring); + if (!IS_GEN2(ring->dev)) { + I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING)); + if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { + DRM_ERROR("%s :timed out trying to stop ring\n", ring->name); + return false; + } + } - /* Stop the ring if it's running. */ I915_WRITE_CTL(ring, 0); I915_WRITE_HEAD(ring, 0); ring->write_tail(ring, 0); - head = I915_READ_HEAD(ring) & HEAD_ADDR; + if (!IS_GEN2(ring->dev)) { + (void)I915_READ_CTL(ring); + I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); + } + + return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0; +} + +static int init_ring_common(struct intel_engine_cs *ring) +{ + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_ringbuffer *ringbuf = ring->buffer; + struct drm_i915_gem_object *obj = ringbuf->obj; + int ret = 0; + + gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL); - /* G45 ring initialization fails to reset head to zero */ - if (head != 0) { + if (!stop_ring(ring)) { + /* G45 ring initialization often fails to reset head to zero */ DRM_DEBUG_KMS("%s head not reset to zero " "ctl %08x head %08x tail %08x start %08x\n", ring->name, @@ -462,9 +499,7 @@ static int init_ring_common(struct intel_ring_buffer *ring) I915_READ_TAIL(ring), I915_READ_START(ring)); - I915_WRITE_HEAD(ring, 0); - - if (I915_READ_HEAD(ring) & HEAD_ADDR) { + if (!stop_ring(ring)) { DRM_ERROR("failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", ring->name, @@ -472,16 +507,23 @@ static int init_ring_common(struct intel_ring_buffer *ring) I915_READ_HEAD(ring), I915_READ_TAIL(ring), I915_READ_START(ring)); + ret = -EIO; + goto out; } } + if (I915_NEED_GFX_HWS(dev)) + intel_ring_setup_status_page(ring); + else + ring_setup_phys_status_page(ring); + /* Initialize the ring. This must happen _after_ we've cleared the ring * registers with the above sequence (the readback of the HEAD registers * also enforces ordering), otherwise the hw might lose the new ring * register values. */ I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj)); I915_WRITE_CTL(ring, - ((ring->size - PAGE_SIZE) & RING_NR_PAGES) + ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID); /* If the head is still not zero, the ring is dead */ @@ -489,12 +531,11 @@ static int init_ring_common(struct intel_ring_buffer *ring) I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) && (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) { DRM_ERROR("%s initialization failed " - "ctl %08x head %08x tail %08x start %08x\n", - ring->name, - I915_READ_CTL(ring), - I915_READ_HEAD(ring), - I915_READ_TAIL(ring), - I915_READ_START(ring)); + "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n", + ring->name, + I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID, + I915_READ_HEAD(ring), I915_READ_TAIL(ring), + I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj)); ret = -EIO; goto out; } @@ -502,10 +543,10 @@ static int init_ring_common(struct intel_ring_buffer *ring) if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) i915_kernel_lost_context(ring->dev); else { - ring->head = I915_READ_HEAD(ring); - ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; - ring->space = ring_space(ring); - ring->last_retired_head = -1; + ringbuf->head = I915_READ_HEAD(ring); + ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR; + ringbuf->space = ring_space(ring); + ringbuf->last_retired_head = -1; } memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); @@ -517,7 +558,7 @@ out: } static int -init_pipe_control(struct intel_ring_buffer *ring) +init_pipe_control(struct intel_engine_cs *ring) { int ret; @@ -531,9 +572,11 @@ init_pipe_control(struct intel_ring_buffer *ring) goto err; } - i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); + ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC); + if (ret) + goto err_unref; - ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, true, false); + ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0); if (ret) goto err_unref; @@ -549,39 +592,42 @@ init_pipe_control(struct intel_ring_buffer *ring) return 0; err_unpin: - i915_gem_object_unpin(ring->scratch.obj); + i915_gem_object_ggtt_unpin(ring->scratch.obj); err_unref: drm_gem_object_unreference(&ring->scratch.obj->base); err: return ret; } -static int init_render_ring(struct intel_ring_buffer *ring) +static int init_render_ring(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; int ret = init_ring_common(ring); - if (INTEL_INFO(dev)->gen > 3) + /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ + if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); /* We need to disable the AsyncFlip performance optimisations in order * to use MI_WAIT_FOR_EVENT within the CS. It should already be * programmed to '1' on all products. * - * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv + * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv */ if (INTEL_INFO(dev)->gen >= 6) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); /* Required for the hardware to program scanline values for waiting */ + /* WaEnableFlushTlbInvalidationMode:snb */ if (INTEL_INFO(dev)->gen == 6) I915_WRITE(GFX_MODE, - _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_ALWAYS)); + _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); + /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ if (IS_GEN7(dev)) I915_WRITE(GFX_MODE_GEN7, - _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) | + _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); if (INTEL_INFO(dev)->gen >= 5) { @@ -598,13 +644,6 @@ static int init_render_ring(struct intel_ring_buffer *ring) */ I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); - - /* This is not explicitly set for GEN6, so read the register. - * see intel_ring_mi_set_context() for why we care. - * TODO: consider explicitly setting the bit for GEN5 - */ - ring->itlb_before_ctx_switch = - !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS); } if (INTEL_INFO(dev)->gen >= 6) @@ -616,7 +655,7 @@ static int init_render_ring(struct intel_ring_buffer *ring) return ret; } -static void render_ring_cleanup(struct intel_ring_buffer *ring) +static void render_ring_cleanup(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; @@ -625,27 +664,53 @@ static void render_ring_cleanup(struct intel_ring_buffer *ring) if (INTEL_INFO(dev)->gen >= 5) { kunmap(sg_page(ring->scratch.obj->pages->sgl)); - i915_gem_object_unpin(ring->scratch.obj); + i915_gem_object_ggtt_unpin(ring->scratch.obj); } drm_gem_object_unreference(&ring->scratch.obj->base); ring->scratch.obj = NULL; } -static void -update_mboxes(struct intel_ring_buffer *ring, - u32 mmio_offset) +static int gen6_signal(struct intel_engine_cs *signaller, + unsigned int num_dwords) { -/* NB: In order to be able to do semaphore MBOX updates for varying number - * of rings, it's easiest if we round up each individual update to a - * multiple of 2 (since ring updates must always be a multiple of 2) - * even though the actual update only requires 3 dwords. - */ + struct drm_device *dev = signaller->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *useless; + int i, ret; + + /* NB: In order to be able to do semaphore MBOX updates for varying + * number of rings, it's easiest if we round up each individual update + * to a multiple of 2 (since ring updates must always be a multiple of + * 2) even though the actual update only requires 3 dwords. + */ #define MBOX_UPDATE_DWORDS 4 - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit(ring, mmio_offset); - intel_ring_emit(ring, ring->outstanding_lazy_seqno); - intel_ring_emit(ring, MI_NOOP); + if (i915_semaphore_is_enabled(dev)) + num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS); + else + return intel_ring_begin(signaller, num_dwords); + + ret = intel_ring_begin(signaller, num_dwords); + if (ret) + return ret; +#undef MBOX_UPDATE_DWORDS + + for_each_ring(useless, dev_priv, i) { + u32 mbox_reg = signaller->semaphore.mbox.signal[i]; + if (mbox_reg != GEN6_NOSYNC) { + intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(signaller, mbox_reg); + intel_ring_emit(signaller, signaller->outstanding_lazy_seqno); + intel_ring_emit(signaller, MI_NOOP); + } else { + intel_ring_emit(signaller, MI_NOOP); + intel_ring_emit(signaller, MI_NOOP); + intel_ring_emit(signaller, MI_NOOP); + intel_ring_emit(signaller, MI_NOOP); + } + } + + return 0; } /** @@ -658,29 +723,14 @@ update_mboxes(struct intel_ring_buffer *ring, * This acts like a signal in the canonical semaphore. */ static int -gen6_add_request(struct intel_ring_buffer *ring) +gen6_add_request(struct intel_engine_cs *ring) { - struct drm_device *dev = ring->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_ring_buffer *useless; - int i, ret, num_dwords = 4; - - if (i915_semaphore_is_enabled(dev)) - num_dwords += ((I915_NUM_RINGS-1) * MBOX_UPDATE_DWORDS); -#undef MBOX_UPDATE_DWORDS + int ret; - ret = intel_ring_begin(ring, num_dwords); + ret = ring->semaphore.signal(ring, 4); if (ret) return ret; - if (i915_semaphore_is_enabled(dev)) { - for_each_ring(useless, dev_priv, i) { - u32 mbox_reg = ring->signal_mbox[i]; - if (mbox_reg != GEN6_NOSYNC) - update_mboxes(ring, mbox_reg); - } - } - intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); intel_ring_emit(ring, ring->outstanding_lazy_seqno); @@ -705,14 +755,15 @@ static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, * @seqno - seqno which the waiter will block on */ static int -gen6_ring_sync(struct intel_ring_buffer *waiter, - struct intel_ring_buffer *signaller, +gen6_ring_sync(struct intel_engine_cs *waiter, + struct intel_engine_cs *signaller, u32 seqno) { - int ret; u32 dw1 = MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER; + u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id]; + int ret; /* Throughout all of the GEM code, seqno passed implies our current * seqno is >= the last seqno executed. However for hardware the @@ -720,8 +771,7 @@ gen6_ring_sync(struct intel_ring_buffer *waiter, */ seqno -= 1; - WARN_ON(signaller->semaphore_register[waiter->id] == - MI_SEMAPHORE_SYNC_INVALID); + WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID); ret = intel_ring_begin(waiter, 4); if (ret) @@ -729,9 +779,7 @@ gen6_ring_sync(struct intel_ring_buffer *waiter, /* If seqno wrap happened, omit the wait with no-ops */ if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) { - intel_ring_emit(waiter, - dw1 | - signaller->semaphore_register[waiter->id]); + intel_ring_emit(waiter, dw1 | wait_mbox); intel_ring_emit(waiter, seqno); intel_ring_emit(waiter, 0); intel_ring_emit(waiter, MI_NOOP); @@ -756,9 +804,9 @@ do { \ } while (0) static int -pc_render_add_request(struct intel_ring_buffer *ring) +pc_render_add_request(struct intel_engine_cs *ring) { - u32 scratch_addr = ring->scratch.gtt_offset + 128; + u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; int ret; /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently @@ -780,15 +828,15 @@ pc_render_add_request(struct intel_ring_buffer *ring) intel_ring_emit(ring, ring->outstanding_lazy_seqno); intel_ring_emit(ring, 0); PIPE_CONTROL_FLUSH(ring, scratch_addr); - scratch_addr += 128; /* write to separate cachelines */ + scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */ PIPE_CONTROL_FLUSH(ring, scratch_addr); - scratch_addr += 128; + scratch_addr += 2 * CACHELINE_BYTES; PIPE_CONTROL_FLUSH(ring, scratch_addr); - scratch_addr += 128; + scratch_addr += 2 * CACHELINE_BYTES; PIPE_CONTROL_FLUSH(ring, scratch_addr); - scratch_addr += 128; + scratch_addr += 2 * CACHELINE_BYTES; PIPE_CONTROL_FLUSH(ring, scratch_addr); - scratch_addr += 128; + scratch_addr += 2 * CACHELINE_BYTES; PIPE_CONTROL_FLUSH(ring, scratch_addr); intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | @@ -804,45 +852,48 @@ pc_render_add_request(struct intel_ring_buffer *ring) } static u32 -gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) +gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) { /* Workaround to force correct ordering between irq and seqno writes on * ivb (and maybe also on snb) by reading from a CS register (like * ACTHD) before reading the status page. */ - if (!lazy_coherency) - intel_ring_get_active_head(ring); + if (!lazy_coherency) { + struct drm_i915_private *dev_priv = ring->dev->dev_private; + POSTING_READ(RING_ACTHD(ring->mmio_base)); + } + return intel_read_status_page(ring, I915_GEM_HWS_INDEX); } static u32 -ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) +ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) { return intel_read_status_page(ring, I915_GEM_HWS_INDEX); } static void -ring_set_seqno(struct intel_ring_buffer *ring, u32 seqno) +ring_set_seqno(struct intel_engine_cs *ring, u32 seqno) { intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); } static u32 -pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency) +pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) { return ring->scratch.cpu_page[0]; } static void -pc_render_set_seqno(struct intel_ring_buffer *ring, u32 seqno) +pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno) { ring->scratch.cpu_page[0] = seqno; } static bool -gen5_ring_get_irq(struct intel_ring_buffer *ring) +gen5_ring_get_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; if (!dev->irq_enabled) @@ -857,10 +908,10 @@ gen5_ring_get_irq(struct intel_ring_buffer *ring) } static void -gen5_ring_put_irq(struct intel_ring_buffer *ring) +gen5_ring_put_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); @@ -870,10 +921,10 @@ gen5_ring_put_irq(struct intel_ring_buffer *ring) } static bool -i9xx_ring_get_irq(struct intel_ring_buffer *ring) +i9xx_ring_get_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; if (!dev->irq_enabled) @@ -891,10 +942,10 @@ i9xx_ring_get_irq(struct intel_ring_buffer *ring) } static void -i9xx_ring_put_irq(struct intel_ring_buffer *ring) +i9xx_ring_put_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); @@ -907,10 +958,10 @@ i9xx_ring_put_irq(struct intel_ring_buffer *ring) } static bool -i8xx_ring_get_irq(struct intel_ring_buffer *ring) +i8xx_ring_get_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; if (!dev->irq_enabled) @@ -928,10 +979,10 @@ i8xx_ring_get_irq(struct intel_ring_buffer *ring) } static void -i8xx_ring_put_irq(struct intel_ring_buffer *ring) +i8xx_ring_put_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); @@ -943,10 +994,10 @@ i8xx_ring_put_irq(struct intel_ring_buffer *ring) spin_unlock_irqrestore(&dev_priv->irq_lock, flags); } -void intel_ring_setup_status_page(struct intel_ring_buffer *ring) +void intel_ring_setup_status_page(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = ring->dev->dev_private; u32 mmio = 0; /* The ring status page addresses are no longer next to the rest of @@ -960,6 +1011,11 @@ void intel_ring_setup_status_page(struct intel_ring_buffer *ring) case BCS: mmio = BLT_HWS_PGA_GEN7; break; + /* + * VCS2 actually doesn't exist on Gen7. Only shut up + * gcc switch check warning + */ + case VCS2: case VCS: mmio = BSD_HWS_PGA_GEN7; break; @@ -977,9 +1033,19 @@ void intel_ring_setup_status_page(struct intel_ring_buffer *ring) I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); POSTING_READ(mmio); - /* Flush the TLB for this page */ - if (INTEL_INFO(dev)->gen >= 6) { + /* + * Flush the TLB for this page + * + * FIXME: These two bits have disappeared on gen8, so a question + * arises: do we still need this and if so how should we go about + * invalidating the TLB? + */ + if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) { u32 reg = RING_INSTPM(ring->mmio_base); + + /* ring should be idle before issuing a sync flush*/ + WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); + I915_WRITE(reg, _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | INSTPM_SYNC_FLUSH)); @@ -991,7 +1057,7 @@ void intel_ring_setup_status_page(struct intel_ring_buffer *ring) } static int -bsd_ring_flush(struct intel_ring_buffer *ring, +bsd_ring_flush(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains) { @@ -1008,7 +1074,7 @@ bsd_ring_flush(struct intel_ring_buffer *ring, } static int -i9xx_add_request(struct intel_ring_buffer *ring) +i9xx_add_request(struct intel_engine_cs *ring) { int ret; @@ -1026,10 +1092,10 @@ i9xx_add_request(struct intel_ring_buffer *ring) } static bool -gen6_ring_get_irq(struct intel_ring_buffer *ring) +gen6_ring_get_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; if (!dev->irq_enabled) @@ -1051,10 +1117,10 @@ gen6_ring_get_irq(struct intel_ring_buffer *ring) } static void -gen6_ring_put_irq(struct intel_ring_buffer *ring) +gen6_ring_put_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; - drm_i915_private_t *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long flags; spin_lock_irqsave(&dev_priv->irq_lock, flags); @@ -1069,7 +1135,7 @@ gen6_ring_put_irq(struct intel_ring_buffer *ring) } static bool -hsw_vebox_get_irq(struct intel_ring_buffer *ring) +hsw_vebox_get_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1089,7 +1155,7 @@ hsw_vebox_get_irq(struct intel_ring_buffer *ring) } static void -hsw_vebox_put_irq(struct intel_ring_buffer *ring) +hsw_vebox_put_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1107,7 +1173,7 @@ hsw_vebox_put_irq(struct intel_ring_buffer *ring) } static bool -gen8_ring_get_irq(struct intel_ring_buffer *ring) +gen8_ring_get_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1133,7 +1199,7 @@ gen8_ring_get_irq(struct intel_ring_buffer *ring) } static void -gen8_ring_put_irq(struct intel_ring_buffer *ring) +gen8_ring_put_irq(struct intel_engine_cs *ring) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1153,8 +1219,8 @@ gen8_ring_put_irq(struct intel_ring_buffer *ring) } static int -i965_dispatch_execbuffer(struct intel_ring_buffer *ring, - u32 offset, u32 length, +i965_dispatch_execbuffer(struct intel_engine_cs *ring, + u64 offset, u32 length, unsigned flags) { int ret; @@ -1176,8 +1242,8 @@ i965_dispatch_execbuffer(struct intel_ring_buffer *ring, /* Just userspace ABI convention to limit the wa batch bo to a resonable size */ #define I830_BATCH_LIMIT (256*1024) static int -i830_dispatch_execbuffer(struct intel_ring_buffer *ring, - u32 offset, u32 len, +i830_dispatch_execbuffer(struct intel_engine_cs *ring, + u64 offset, u32 len, unsigned flags) { int ret; @@ -1227,8 +1293,8 @@ i830_dispatch_execbuffer(struct intel_ring_buffer *ring, } static int -i915_dispatch_execbuffer(struct intel_ring_buffer *ring, - u32 offset, u32 len, +i915_dispatch_execbuffer(struct intel_engine_cs *ring, + u64 offset, u32 len, unsigned flags) { int ret; @@ -1244,7 +1310,7 @@ i915_dispatch_execbuffer(struct intel_ring_buffer *ring, return 0; } -static void cleanup_status_page(struct intel_ring_buffer *ring) +static void cleanup_status_page(struct intel_engine_cs *ring) { struct drm_i915_gem_object *obj; @@ -1253,54 +1319,49 @@ static void cleanup_status_page(struct intel_ring_buffer *ring) return; kunmap(sg_page(obj->pages->sgl)); - i915_gem_object_unpin(obj); + i915_gem_object_ggtt_unpin(obj); drm_gem_object_unreference(&obj->base); ring->status_page.obj = NULL; } -static int init_status_page(struct intel_ring_buffer *ring) +static int init_status_page(struct intel_engine_cs *ring) { - struct drm_device *dev = ring->dev; struct drm_i915_gem_object *obj; - int ret; - obj = i915_gem_alloc_object(dev, 4096); - if (obj == NULL) { - DRM_ERROR("Failed to allocate status page\n"); - ret = -ENOMEM; - goto err; - } + if ((obj = ring->status_page.obj) == NULL) { + int ret; + + obj = i915_gem_alloc_object(ring->dev, 4096); + if (obj == NULL) { + DRM_ERROR("Failed to allocate status page\n"); + return -ENOMEM; + } - i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); + ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); + if (ret) + goto err_unref; - ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false); - if (ret != 0) { - goto err_unref; + ret = i915_gem_obj_ggtt_pin(obj, 4096, 0); + if (ret) { +err_unref: + drm_gem_object_unreference(&obj->base); + return ret; + } + + ring->status_page.obj = obj; } ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj); ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl)); - if (ring->status_page.page_addr == NULL) { - ret = -ENOMEM; - goto err_unpin; - } - ring->status_page.obj = obj; memset(ring->status_page.page_addr, 0, PAGE_SIZE); DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", ring->name, ring->status_page.gfx_addr); return 0; - -err_unpin: - i915_gem_object_unpin(obj); -err_unref: - drm_gem_object_unreference(&obj->base); -err: - return ret; } -static int init_phys_status_page(struct intel_ring_buffer *ring) +static int init_phys_status_page(struct intel_engine_cs *ring) { struct drm_i915_private *dev_priv = ring->dev->dev_private; @@ -1317,46 +1378,26 @@ static int init_phys_status_page(struct intel_ring_buffer *ring) return 0; } -static int intel_init_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring) +static int allocate_ring_buffer(struct intel_engine_cs *ring) { + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_ringbuffer *ringbuf = ring->buffer; struct drm_i915_gem_object *obj; - struct drm_i915_private *dev_priv = dev->dev_private; int ret; - ring->dev = dev; - INIT_LIST_HEAD(&ring->active_list); - INIT_LIST_HEAD(&ring->request_list); - ring->size = 32 * PAGE_SIZE; - memset(ring->sync_seqno, 0, sizeof(ring->sync_seqno)); - - init_waitqueue_head(&ring->irq_queue); - - if (I915_NEED_GFX_HWS(dev)) { - ret = init_status_page(ring); - if (ret) - return ret; - } else { - BUG_ON(ring->id != RCS); - ret = init_phys_status_page(ring); - if (ret) - return ret; - } + if (intel_ring_initialized(ring)) + return 0; obj = NULL; if (!HAS_LLC(dev)) - obj = i915_gem_object_create_stolen(dev, ring->size); + obj = i915_gem_object_create_stolen(dev, ringbuf->size); if (obj == NULL) - obj = i915_gem_alloc_object(dev, ring->size); - if (obj == NULL) { - DRM_ERROR("Failed to allocate ringbuffer\n"); - ret = -ENOMEM; - goto err_hws; - } - - ring->obj = obj; + obj = i915_gem_alloc_object(dev, ringbuf->size); + if (obj == NULL) + return -ENOMEM; - ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, true, false); + ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE); if (ret) goto err_unref; @@ -1364,63 +1405,102 @@ static int intel_init_ring_buffer(struct drm_device *dev, if (ret) goto err_unpin; - ring->virtual_start = + ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj), - ring->size); - if (ring->virtual_start == NULL) { - DRM_ERROR("Failed to map ringbuffer.\n"); + ringbuf->size); + if (ringbuf->virtual_start == NULL) { ret = -EINVAL; goto err_unpin; } - ret = ring->init(ring); - if (ret) - goto err_unmap; + ringbuf->obj = obj; + return 0; + +err_unpin: + i915_gem_object_ggtt_unpin(obj); +err_unref: + drm_gem_object_unreference(&obj->base); + return ret; +} + +static int intel_init_ring_buffer(struct drm_device *dev, + struct intel_engine_cs *ring) +{ + struct intel_ringbuffer *ringbuf = ring->buffer; + int ret; + + if (ringbuf == NULL) { + ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); + if (!ringbuf) + return -ENOMEM; + ring->buffer = ringbuf; + } + + ring->dev = dev; + INIT_LIST_HEAD(&ring->active_list); + INIT_LIST_HEAD(&ring->request_list); + ringbuf->size = 32 * PAGE_SIZE; + memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno)); + + init_waitqueue_head(&ring->irq_queue); + + if (I915_NEED_GFX_HWS(dev)) { + ret = init_status_page(ring); + if (ret) + goto error; + } else { + BUG_ON(ring->id != RCS); + ret = init_phys_status_page(ring); + if (ret) + goto error; + } + + ret = allocate_ring_buffer(ring); + if (ret) { + DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret); + goto error; + } /* Workaround an erratum on the i830 which causes a hang if * the TAIL pointer points to within the last 2 cachelines * of the buffer. */ - ring->effective_size = ring->size; - if (IS_I830(ring->dev) || IS_845G(ring->dev)) - ring->effective_size -= 128; + ringbuf->effective_size = ringbuf->size; + if (IS_I830(dev) || IS_845G(dev)) + ringbuf->effective_size -= 2 * CACHELINE_BYTES; + + ret = i915_cmd_parser_init_ring(ring); + if (ret) + goto error; + + ret = ring->init(ring); + if (ret) + goto error; return 0; -err_unmap: - iounmap(ring->virtual_start); -err_unpin: - i915_gem_object_unpin(obj); -err_unref: - drm_gem_object_unreference(&obj->base); - ring->obj = NULL; -err_hws: - cleanup_status_page(ring); +error: + kfree(ringbuf); + ring->buffer = NULL; return ret; } -void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) +void intel_cleanup_ring_buffer(struct intel_engine_cs *ring) { - struct drm_i915_private *dev_priv; - int ret; + struct drm_i915_private *dev_priv = to_i915(ring->dev); + struct intel_ringbuffer *ringbuf = ring->buffer; - if (ring->obj == NULL) + if (!intel_ring_initialized(ring)) return; - /* Disable the ring buffer. The ring must be idle at this point */ - dev_priv = ring->dev->dev_private; - ret = intel_ring_idle(ring); - if (ret && !i915_reset_in_progress(&dev_priv->gpu_error)) - DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n", - ring->name, ret); + intel_stop_ring_buffer(ring); + WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0); - I915_WRITE_CTL(ring, 0); + iounmap(ringbuf->virtual_start); - iounmap(ring->virtual_start); - - i915_gem_object_unpin(ring->obj); - drm_gem_object_unreference(&ring->obj->base); - ring->obj = NULL; + i915_gem_object_ggtt_unpin(ringbuf->obj); + drm_gem_object_unreference(&ringbuf->obj->base); + ringbuf->obj = NULL; ring->preallocated_lazy_request = NULL; ring->outstanding_lazy_seqno = 0; @@ -1428,80 +1508,56 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) ring->cleanup(ring); cleanup_status_page(ring); -} -static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno) -{ - int ret; + i915_cmd_parser_fini_ring(ring); - ret = i915_wait_seqno(ring, seqno); - if (!ret) - i915_gem_retire_requests_ring(ring); - - return ret; + kfree(ringbuf); + ring->buffer = NULL; } -static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n) +static int intel_ring_wait_request(struct intel_engine_cs *ring, int n) { + struct intel_ringbuffer *ringbuf = ring->buffer; struct drm_i915_gem_request *request; u32 seqno = 0; int ret; - i915_gem_retire_requests_ring(ring); + if (ringbuf->last_retired_head != -1) { + ringbuf->head = ringbuf->last_retired_head; + ringbuf->last_retired_head = -1; - if (ring->last_retired_head != -1) { - ring->head = ring->last_retired_head; - ring->last_retired_head = -1; - ring->space = ring_space(ring); - if (ring->space >= n) + ringbuf->space = ring_space(ring); + if (ringbuf->space >= n) return 0; } list_for_each_entry(request, &ring->request_list, list) { - int space; - - if (request->tail == -1) - continue; - - space = request->tail - (ring->tail + I915_RING_FREE_SPACE); - if (space < 0) - space += ring->size; - if (space >= n) { + if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) { seqno = request->seqno; break; } - - /* Consume this request in case we need more space than - * is available and so need to prevent a race between - * updating last_retired_head and direct reads of - * I915_RING_HEAD. It also provides a nice sanity check. - */ - request->tail = -1; } if (seqno == 0) return -ENOSPC; - ret = intel_ring_wait_seqno(ring, seqno); + ret = i915_wait_seqno(ring, seqno); if (ret) return ret; - if (WARN_ON(ring->last_retired_head == -1)) - return -ENOSPC; - - ring->head = ring->last_retired_head; - ring->last_retired_head = -1; - ring->space = ring_space(ring); - if (WARN_ON(ring->space < n)) - return -ENOSPC; + i915_gem_retire_requests_ring(ring); + ringbuf->head = ringbuf->last_retired_head; + ringbuf->last_retired_head = -1; + ringbuf->space = ring_space(ring); return 0; } -static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) +static int ring_wait_for_space(struct intel_engine_cs *ring, int n) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_ringbuffer *ringbuf = ring->buffer; unsigned long end; int ret; @@ -1512,7 +1568,6 @@ static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) /* force the tail write in case we have been skipping them */ __intel_ring_advance(ring); - trace_i915_ring_wait_begin(ring); /* With GEM the hangcheck timer should kick us out of the loop, * leaving it early runs the risk of corrupting GEM state (due * to running on almost untested codepaths). But on resume @@ -1520,15 +1575,17 @@ static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) * case by choosing an insanely large timeout. */ end = jiffies + 60 * HZ; + trace_i915_ring_wait_begin(ring); do { - ring->head = I915_READ_HEAD(ring); - ring->space = ring_space(ring); - if (ring->space >= n) { - trace_i915_ring_wait_end(ring); - return 0; + ringbuf->head = I915_READ_HEAD(ring); + ringbuf->space = ring_space(ring); + if (ringbuf->space >= n) { + ret = 0; + break; } - if (dev->primary->master) { + if (!drm_core_check_feature(dev, DRIVER_MODESET) && + dev->primary->master) { struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; if (master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; @@ -1536,38 +1593,49 @@ static int ring_wait_for_space(struct intel_ring_buffer *ring, int n) msleep(1); + if (dev_priv->mm.interruptible && signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + ret = i915_gem_check_wedge(&dev_priv->gpu_error, dev_priv->mm.interruptible); if (ret) - return ret; - } while (!time_after(jiffies, end)); + break; + + if (time_after(jiffies, end)) { + ret = -EBUSY; + break; + } + } while (1); trace_i915_ring_wait_end(ring); - return -EBUSY; + return ret; } -static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring) +static int intel_wrap_ring_buffer(struct intel_engine_cs *ring) { uint32_t __iomem *virt; - int rem = ring->size - ring->tail; + struct intel_ringbuffer *ringbuf = ring->buffer; + int rem = ringbuf->size - ringbuf->tail; - if (ring->space < rem) { + if (ringbuf->space < rem) { int ret = ring_wait_for_space(ring, rem); if (ret) return ret; } - virt = ring->virtual_start + ring->tail; + virt = ringbuf->virtual_start + ringbuf->tail; rem /= 4; while (rem--) iowrite32(MI_NOOP, virt++); - ring->tail = 0; - ring->space = ring_space(ring); + ringbuf->tail = 0; + ringbuf->space = ring_space(ring); return 0; } -int intel_ring_idle(struct intel_ring_buffer *ring) +int intel_ring_idle(struct intel_engine_cs *ring) { u32 seqno; int ret; @@ -1591,7 +1659,7 @@ int intel_ring_idle(struct intel_ring_buffer *ring) } static int -intel_ring_alloc_seqno(struct intel_ring_buffer *ring) +intel_ring_alloc_seqno(struct intel_engine_cs *ring) { if (ring->outstanding_lazy_seqno) return 0; @@ -1609,18 +1677,19 @@ intel_ring_alloc_seqno(struct intel_ring_buffer *ring) return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno); } -static int __intel_ring_prepare(struct intel_ring_buffer *ring, +static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes) { + struct intel_ringbuffer *ringbuf = ring->buffer; int ret; - if (unlikely(ring->tail + bytes > ring->effective_size)) { + if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) { ret = intel_wrap_ring_buffer(ring); if (unlikely(ret)) return ret; } - if (unlikely(ring->space < bytes)) { + if (unlikely(ringbuf->space < bytes)) { ret = ring_wait_for_space(ring, bytes); if (unlikely(ret)) return ret; @@ -1629,10 +1698,10 @@ static int __intel_ring_prepare(struct intel_ring_buffer *ring, return 0; } -int intel_ring_begin(struct intel_ring_buffer *ring, +int intel_ring_begin(struct intel_engine_cs *ring, int num_dwords) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = ring->dev->dev_private; int ret; ret = i915_gem_check_wedge(&dev_priv->gpu_error, @@ -1649,11 +1718,33 @@ int intel_ring_begin(struct intel_ring_buffer *ring, if (ret) return ret; - ring->space -= num_dwords * sizeof(uint32_t); + ring->buffer->space -= num_dwords * sizeof(uint32_t); return 0; } -void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno) +/* Align the ring tail to a cacheline boundary */ +int intel_ring_cacheline_align(struct intel_engine_cs *ring) +{ + int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t); + int ret; + + if (num_dwords == 0) + return 0; + + num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords; + ret = intel_ring_begin(ring, num_dwords); + if (ret) + return ret; + + while (num_dwords--) + intel_ring_emit(ring, MI_NOOP); + + intel_ring_advance(ring); + + return 0; +} + +void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno) { struct drm_i915_private *dev_priv = ring->dev->dev_private; @@ -1670,10 +1761,10 @@ void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno) ring->hangcheck.seqno = seqno; } -static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, +static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring, u32 value) { - drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_private *dev_priv = ring->dev->dev_private; /* Every tail move must follow the sequence below */ @@ -1703,7 +1794,7 @@ static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); } -static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring, +static int gen6_bsd_ring_flush(struct intel_engine_cs *ring, u32 invalidate, u32 flush) { uint32_t cmd; @@ -1739,8 +1830,8 @@ static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring, } static int -gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, - u32 offset, u32 len, +gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring, + u64 offset, u32 len, unsigned flags) { struct drm_i915_private *dev_priv = ring->dev->dev_private; @@ -1754,8 +1845,8 @@ gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, /* FIXME(BDW): Address space and security selectors. */ intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); - intel_ring_emit(ring, offset); - intel_ring_emit(ring, 0); + intel_ring_emit(ring, lower_32_bits(offset)); + intel_ring_emit(ring, upper_32_bits(offset)); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); @@ -1763,8 +1854,8 @@ gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, } static int -hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, - u32 offset, u32 len, +hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring, + u64 offset, u32 len, unsigned flags) { int ret; @@ -1784,8 +1875,8 @@ hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, } static int -gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, - u32 offset, u32 len, +gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring, + u64 offset, u32 len, unsigned flags) { int ret; @@ -1806,7 +1897,7 @@ gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, /* Blitter support (SandyBridge+) */ -static int gen6_ring_flush(struct intel_ring_buffer *ring, +static int gen6_ring_flush(struct intel_engine_cs *ring, u32 invalidate, u32 flush) { struct drm_device *dev = ring->dev; @@ -1848,8 +1939,8 @@ static int gen6_ring_flush(struct intel_ring_buffer *ring, int intel_init_render_ring_buffer(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; ring->name = "render ring"; ring->id = RCS; @@ -1871,15 +1962,24 @@ int intel_init_render_ring_buffer(struct drm_device *dev) ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT; ring->get_seqno = gen6_ring_get_seqno; ring->set_seqno = ring_set_seqno; - ring->sync_to = gen6_ring_sync; - ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV; - ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB; - ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_RVE; - ring->signal_mbox[RCS] = GEN6_NOSYNC; - ring->signal_mbox[VCS] = GEN6_VRSYNC; - ring->signal_mbox[BCS] = GEN6_BRSYNC; - ring->signal_mbox[VECS] = GEN6_VERSYNC; + ring->semaphore.sync_to = gen6_ring_sync; + ring->semaphore.signal = gen6_signal; + /* + * The current semaphore is only applied on pre-gen8 platform. + * And there is no VCS2 ring on the pre-gen8 platform. So the + * semaphore between RCS and VCS2 is initialized as INVALID. + * Gen8 will initialize the sema between VCS2 and RCS later. + */ + ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; + ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; + ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; + ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; + ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; + ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; + ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; } else if (IS_GEN5(dev)) { ring->add_request = pc_render_add_request; ring->flush = gen4_render_ring_flush; @@ -1933,7 +2033,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev) return -ENOMEM; } - ret = i915_gem_obj_ggtt_pin(obj, 0, true, false); + ret = i915_gem_obj_ggtt_pin(obj, 0, 0); if (ret != 0) { drm_gem_object_unreference(&obj->base); DRM_ERROR("Failed to ping batch bo\n"); @@ -1949,17 +2049,26 @@ int intel_init_render_ring_buffer(struct drm_device *dev) int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[RCS]; + struct intel_ringbuffer *ringbuf = ring->buffer; int ret; + if (ringbuf == NULL) { + ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); + if (!ringbuf) + return -ENOMEM; + ring->buffer = ringbuf; + } + ring->name = "render ring"; ring->id = RCS; ring->mmio_base = RENDER_RING_BASE; if (INTEL_INFO(dev)->gen >= 6) { /* non-kms not supported on gen6+ */ - return -ENODEV; + ret = -ENODEV; + goto err_ringbuf; } /* Note: gem is not supported on gen5/ilk without kms (the corresponding @@ -1994,31 +2103,39 @@ int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size) INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); - ring->size = size; - ring->effective_size = ring->size; + ringbuf->size = size; + ringbuf->effective_size = ringbuf->size; if (IS_I830(ring->dev) || IS_845G(ring->dev)) - ring->effective_size -= 128; + ringbuf->effective_size -= 2 * CACHELINE_BYTES; - ring->virtual_start = ioremap_wc(start, size); - if (ring->virtual_start == NULL) { + ringbuf->virtual_start = ioremap_wc(start, size); + if (ringbuf->virtual_start == NULL) { DRM_ERROR("can not ioremap virtual address for" " ring buffer\n"); - return -ENOMEM; + ret = -ENOMEM; + goto err_ringbuf; } if (!I915_NEED_GFX_HWS(dev)) { ret = init_phys_status_page(ring); if (ret) - return ret; + goto err_vstart; } return 0; + +err_vstart: + iounmap(ringbuf->virtual_start); +err_ringbuf: + kfree(ringbuf); + ring->buffer = NULL; + return ret; } int intel_init_bsd_ring_buffer(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[VCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[VCS]; ring->name = "bsd ring"; ring->id = VCS; @@ -2047,15 +2164,24 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev) ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; } - ring->sync_to = gen6_ring_sync; - ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR; - ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB; - ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_VVE; - ring->signal_mbox[RCS] = GEN6_RVSYNC; - ring->signal_mbox[VCS] = GEN6_NOSYNC; - ring->signal_mbox[BCS] = GEN6_BVSYNC; - ring->signal_mbox[VECS] = GEN6_VEVSYNC; + ring->semaphore.sync_to = gen6_ring_sync; + ring->semaphore.signal = gen6_signal; + /* + * The current semaphore is only applied on pre-gen8 platform. + * And there is no VCS2 ring on the pre-gen8 platform. So the + * semaphore between VCS and VCS2 is initialized as INVALID. + * Gen8 will initialize the sema between VCS2 and VCS later. + */ + ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; + ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; + ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; + ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; + ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; + ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; + ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; } else { ring->mmio_base = BSD_RING_BASE; ring->flush = bsd_ring_flush; @@ -2078,10 +2204,63 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev) return intel_init_ring_buffer(dev, ring); } +/** + * Initialize the second BSD ring for Broadwell GT3. + * It is noted that this only exists on Broadwell GT3. + */ +int intel_init_bsd2_ring_buffer(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[VCS2]; + + if ((INTEL_INFO(dev)->gen != 8)) { + DRM_ERROR("No dual-BSD ring on non-BDW machine\n"); + return -EINVAL; + } + + ring->name = "bds2_ring"; + ring->id = VCS2; + + ring->write_tail = ring_write_tail; + ring->mmio_base = GEN8_BSD2_RING_BASE; + ring->flush = gen6_bsd_ring_flush; + ring->add_request = gen6_add_request; + ring->get_seqno = gen6_ring_get_seqno; + ring->set_seqno = ring_set_seqno; + ring->irq_enable_mask = + GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; + ring->irq_get = gen8_ring_get_irq; + ring->irq_put = gen8_ring_put_irq; + ring->dispatch_execbuffer = + gen8_ring_dispatch_execbuffer; + ring->semaphore.sync_to = gen6_ring_sync; + ring->semaphore.signal = gen6_signal; + /* + * The current semaphore is only applied on the pre-gen8. And there + * is no bsd2 ring on the pre-gen8. So now the semaphore_register + * between VCS2 and other ring is initialized as invalid. + * Gen8 will initialize the sema between VCS2 and other ring later. + */ + ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; + + ring->init = init_ring_common; + + return intel_init_ring_buffer(dev, ring); +} + int intel_init_blt_ring_buffer(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[BCS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[BCS]; ring->name = "blitter ring"; ring->id = BCS; @@ -2104,15 +2283,24 @@ int intel_init_blt_ring_buffer(struct drm_device *dev) ring->irq_put = gen6_ring_put_irq; ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; } - ring->sync_to = gen6_ring_sync; - ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR; - ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV; - ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID; - ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_BVE; - ring->signal_mbox[RCS] = GEN6_RBSYNC; - ring->signal_mbox[VCS] = GEN6_VBSYNC; - ring->signal_mbox[BCS] = GEN6_NOSYNC; - ring->signal_mbox[VECS] = GEN6_VEBSYNC; + ring->semaphore.sync_to = gen6_ring_sync; + ring->semaphore.signal = gen6_signal; + /* + * The current semaphore is only applied on pre-gen8 platform. And + * there is no VCS2 ring on the pre-gen8 platform. So the semaphore + * between BCS and VCS2 is initialized as INVALID. + * Gen8 will initialize the sema between BCS and VCS2 later. + */ + ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; + ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; + ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; + ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; + ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; + ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; + ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; ring->init = init_ring_common; return intel_init_ring_buffer(dev, ring); @@ -2120,8 +2308,8 @@ int intel_init_blt_ring_buffer(struct drm_device *dev) int intel_init_vebox_ring_buffer(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct intel_ring_buffer *ring = &dev_priv->ring[VECS]; + struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_engine_cs *ring = &dev_priv->ring[VECS]; ring->name = "video enhancement ring"; ring->id = VECS; @@ -2145,22 +2333,25 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev) ring->irq_put = hsw_vebox_put_irq; ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; } - ring->sync_to = gen6_ring_sync; - ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER; - ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV; - ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB; - ring->semaphore_register[VECS] = MI_SEMAPHORE_SYNC_INVALID; - ring->signal_mbox[RCS] = GEN6_RVESYNC; - ring->signal_mbox[VCS] = GEN6_VVESYNC; - ring->signal_mbox[BCS] = GEN6_BVESYNC; - ring->signal_mbox[VECS] = GEN6_NOSYNC; + ring->semaphore.sync_to = gen6_ring_sync; + ring->semaphore.signal = gen6_signal; + ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; + ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; + ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; + ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; + ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; + ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; + ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; + ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; + ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; ring->init = init_ring_common; return intel_init_ring_buffer(dev, ring); } int -intel_ring_flush_all_caches(struct intel_ring_buffer *ring) +intel_ring_flush_all_caches(struct intel_engine_cs *ring) { int ret; @@ -2178,7 +2369,7 @@ intel_ring_flush_all_caches(struct intel_ring_buffer *ring) } int -intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring) +intel_ring_invalidate_all_caches(struct intel_engine_cs *ring) { uint32_t flush_domains; int ret; @@ -2196,3 +2387,19 @@ intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring) ring->gpu_caches_dirty = false; return 0; } + +void +intel_stop_ring_buffer(struct intel_engine_cs *ring) +{ + int ret; + + if (!intel_ring_initialized(ring)) + return; + + ret = intel_ring_idle(ring); + if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error)) + DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n", + ring->name, ret); + + stop_ring(ring); +} diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 71a73f4fe25..e72017bdcd7 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -1,6 +1,10 @@ #ifndef _INTEL_RINGBUFFER_H_ #define _INTEL_RINGBUFFER_H_ +#include <linux/hashtable.h> + +#define I915_CMD_HASH_ORDER 9 + /* * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use" * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use" @@ -33,6 +37,9 @@ struct intel_hw_status_page { #define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base)) #define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val) +#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base)) +#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val) + enum intel_ring_hangcheck_action { HANGCHECK_IDLE = 0, HANGCHECK_WAIT, @@ -41,84 +48,103 @@ enum intel_ring_hangcheck_action { HANGCHECK_HUNG, }; +#define HANGCHECK_SCORE_RING_HUNG 31 + struct intel_ring_hangcheck { - bool deadlock; + u64 acthd; u32 seqno; - u32 acthd; int score; enum intel_ring_hangcheck_action action; + int deadlock; }; -struct intel_ring_buffer { +struct intel_ringbuffer { + struct drm_i915_gem_object *obj; + void __iomem *virtual_start; + + u32 head; + u32 tail; + int space; + int size; + int effective_size; + + /** We track the position of the requests in the ring buffer, and + * when each is retired we increment last_retired_head as the GPU + * must have finished processing the request and so we know we + * can advance the ringbuffer up to that position. + * + * last_retired_head is set to -1 after the value is consumed so + * we can detect new retirements. + */ + u32 last_retired_head; +}; + +struct intel_engine_cs { const char *name; enum intel_ring_id { RCS = 0x0, VCS, BCS, VECS, + VCS2 } id; -#define I915_NUM_RINGS 4 +#define I915_NUM_RINGS 5 +#define LAST_USER_RING (VECS + 1) u32 mmio_base; - void __iomem *virtual_start; struct drm_device *dev; - struct drm_i915_gem_object *obj; + struct intel_ringbuffer *buffer; - u32 head; - u32 tail; - int space; - int size; - int effective_size; struct intel_hw_status_page status_page; - /** We track the position of the requests in the ring buffer, and - * when each is retired we increment last_retired_head as the GPU - * must have finished processing the request and so we know we - * can advance the ringbuffer up to that position. - * - * last_retired_head is set to -1 after the value is consumed so - * we can detect new retirements. - */ - u32 last_retired_head; - unsigned irq_refcount; /* protected by dev_priv->irq_lock */ u32 irq_enable_mask; /* bitmask to enable ring interrupt */ u32 trace_irq_seqno; - u32 sync_seqno[I915_NUM_RINGS-1]; - bool __must_check (*irq_get)(struct intel_ring_buffer *ring); - void (*irq_put)(struct intel_ring_buffer *ring); + bool __must_check (*irq_get)(struct intel_engine_cs *ring); + void (*irq_put)(struct intel_engine_cs *ring); - int (*init)(struct intel_ring_buffer *ring); + int (*init)(struct intel_engine_cs *ring); - void (*write_tail)(struct intel_ring_buffer *ring, + void (*write_tail)(struct intel_engine_cs *ring, u32 value); - int __must_check (*flush)(struct intel_ring_buffer *ring, + int __must_check (*flush)(struct intel_engine_cs *ring, u32 invalidate_domains, u32 flush_domains); - int (*add_request)(struct intel_ring_buffer *ring); + int (*add_request)(struct intel_engine_cs *ring); /* Some chipsets are not quite as coherent as advertised and need * an expensive kick to force a true read of the up-to-date seqno. * However, the up-to-date seqno is not always required and the last * seen value is good enough. Note that the seqno will always be * monotonic, even if not coherent. */ - u32 (*get_seqno)(struct intel_ring_buffer *ring, + u32 (*get_seqno)(struct intel_engine_cs *ring, bool lazy_coherency); - void (*set_seqno)(struct intel_ring_buffer *ring, + void (*set_seqno)(struct intel_engine_cs *ring, u32 seqno); - int (*dispatch_execbuffer)(struct intel_ring_buffer *ring, - u32 offset, u32 length, + int (*dispatch_execbuffer)(struct intel_engine_cs *ring, + u64 offset, u32 length, unsigned flags); #define I915_DISPATCH_SECURE 0x1 #define I915_DISPATCH_PINNED 0x2 - void (*cleanup)(struct intel_ring_buffer *ring); - int (*sync_to)(struct intel_ring_buffer *ring, - struct intel_ring_buffer *to, - u32 seqno); + void (*cleanup)(struct intel_engine_cs *ring); - /* our mbox written by others */ - u32 semaphore_register[I915_NUM_RINGS]; - /* mboxes this ring signals to */ - u32 signal_mbox[I915_NUM_RINGS]; + struct { + u32 sync_seqno[I915_NUM_RINGS-1]; + + struct { + /* our mbox written by others */ + u32 wait[I915_NUM_RINGS]; + /* mboxes this ring signals to */ + u32 signal[I915_NUM_RINGS]; + } mbox; + + /* AKA wait() */ + int (*sync_to)(struct intel_engine_cs *ring, + struct intel_engine_cs *to, + u32 seqno); + int (*signal)(struct intel_engine_cs *signaller, + /* num_dwords needed by caller */ + unsigned int num_dwords); + } semaphore; /** * List of objects currently involved in rendering from the @@ -148,12 +174,8 @@ struct intel_ring_buffer { wait_queue_head_t irq_queue; - /** - * Do an explicit TLB flush before MI_SET_CONTEXT - */ - bool itlb_before_ctx_switch; - struct i915_hw_context *default_context; - struct i915_hw_context *last_context; + struct intel_context *default_context; + struct intel_context *last_context; struct intel_ring_hangcheck hangcheck; @@ -162,23 +184,56 @@ struct intel_ring_buffer { u32 gtt_offset; volatile u32 *cpu_page; } scratch; + + bool needs_cmd_parser; + + /* + * Table of commands the command parser needs to know about + * for this ring. + */ + DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER); + + /* + * Table of registers allowed in commands that read/write registers. + */ + const u32 *reg_table; + int reg_count; + + /* + * Table of registers allowed in commands that read/write registers, but + * only from the DRM master. + */ + const u32 *master_reg_table; + int master_reg_count; + + /* + * Returns the bitmask for the length field of the specified command. + * Return 0 for an unrecognized/invalid command. + * + * If the command parser finds an entry for a command in the ring's + * cmd_tables, it gets the command's length based on the table entry. + * If not, it calls this function to determine the per-ring length field + * encoding for the command (i.e. certain opcode ranges use certain bits + * to encode the command length in the header). + */ + u32 (*get_cmd_length_mask)(u32 cmd_header); }; static inline bool -intel_ring_initialized(struct intel_ring_buffer *ring) +intel_ring_initialized(struct intel_engine_cs *ring) { - return ring->obj != NULL; + return ring->buffer && ring->buffer->obj; } static inline unsigned -intel_ring_flag(struct intel_ring_buffer *ring) +intel_ring_flag(struct intel_engine_cs *ring) { return 1 << ring->id; } static inline u32 -intel_ring_sync_index(struct intel_ring_buffer *ring, - struct intel_ring_buffer *other) +intel_ring_sync_index(struct intel_engine_cs *ring, + struct intel_engine_cs *other) { int idx; @@ -196,7 +251,7 @@ intel_ring_sync_index(struct intel_ring_buffer *ring, } static inline u32 -intel_read_status_page(struct intel_ring_buffer *ring, +intel_read_status_page(struct intel_engine_cs *ring, int reg) { /* Ensure that the compiler doesn't optimize away the load. */ @@ -205,7 +260,7 @@ intel_read_status_page(struct intel_ring_buffer *ring, } static inline void -intel_write_status_page(struct intel_ring_buffer *ring, +intel_write_status_page(struct intel_engine_cs *ring, int reg, u32 value) { ring->status_page.page_addr[reg] = value; @@ -230,46 +285,51 @@ intel_write_status_page(struct intel_ring_buffer *ring, #define I915_GEM_HWS_SCRATCH_INDEX 0x30 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT) -void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring); +void intel_stop_ring_buffer(struct intel_engine_cs *ring); +void intel_cleanup_ring_buffer(struct intel_engine_cs *ring); -int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n); -static inline void intel_ring_emit(struct intel_ring_buffer *ring, +int __must_check intel_ring_begin(struct intel_engine_cs *ring, int n); +int __must_check intel_ring_cacheline_align(struct intel_engine_cs *ring); +static inline void intel_ring_emit(struct intel_engine_cs *ring, u32 data) { - iowrite32(data, ring->virtual_start + ring->tail); - ring->tail += 4; + struct intel_ringbuffer *ringbuf = ring->buffer; + iowrite32(data, ringbuf->virtual_start + ringbuf->tail); + ringbuf->tail += 4; } -static inline void intel_ring_advance(struct intel_ring_buffer *ring) +static inline void intel_ring_advance(struct intel_engine_cs *ring) { - ring->tail &= ring->size - 1; + struct intel_ringbuffer *ringbuf = ring->buffer; + ringbuf->tail &= ringbuf->size - 1; } -void __intel_ring_advance(struct intel_ring_buffer *ring); +void __intel_ring_advance(struct intel_engine_cs *ring); -int __must_check intel_ring_idle(struct intel_ring_buffer *ring); -void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno); -int intel_ring_flush_all_caches(struct intel_ring_buffer *ring); -int intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring); +int __must_check intel_ring_idle(struct intel_engine_cs *ring); +void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno); +int intel_ring_flush_all_caches(struct intel_engine_cs *ring); +int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring); int intel_init_render_ring_buffer(struct drm_device *dev); int intel_init_bsd_ring_buffer(struct drm_device *dev); +int intel_init_bsd2_ring_buffer(struct drm_device *dev); int intel_init_blt_ring_buffer(struct drm_device *dev); int intel_init_vebox_ring_buffer(struct drm_device *dev); -u32 intel_ring_get_active_head(struct intel_ring_buffer *ring); -void intel_ring_setup_status_page(struct intel_ring_buffer *ring); +u64 intel_ring_get_active_head(struct intel_engine_cs *ring); +void intel_ring_setup_status_page(struct intel_engine_cs *ring); -static inline u32 intel_ring_get_tail(struct intel_ring_buffer *ring) +static inline u32 intel_ring_get_tail(struct intel_engine_cs *ring) { - return ring->tail; + return ring->buffer->tail; } -static inline u32 intel_ring_get_seqno(struct intel_ring_buffer *ring) +static inline u32 intel_ring_get_seqno(struct intel_engine_cs *ring) { BUG_ON(ring->outstanding_lazy_seqno == 0); return ring->outstanding_lazy_seqno; } -static inline void i915_trace_irq_get(struct intel_ring_buffer *ring, u32 seqno) +static inline void i915_trace_irq_get(struct intel_engine_cs *ring, u32 seqno) { if (ring->trace_irq_seqno == 0 && ring->irq_get(ring)) ring->trace_irq_seqno = seqno; diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 95bdfb3c431..20375cc7f82 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -1153,20 +1153,21 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, pipe_config->pixel_multiplier = intel_sdvo_get_pixel_multiplier(adjusted_mode); + pipe_config->has_hdmi_sink = intel_sdvo->has_hdmi_monitor; + if (intel_sdvo->color_range_auto) { /* See CEA-861-E - 5.1 Default Encoding Parameters */ /* FIXME: This bit is only valid when using TMDS encoding and 8 * bit per color mode. */ - if (intel_sdvo->has_hdmi_monitor && + if (pipe_config->has_hdmi_sink && drm_match_cea_mode(adjusted_mode) > 1) - intel_sdvo->color_range = HDMI_COLOR_RANGE_16_235; - else - intel_sdvo->color_range = 0; + pipe_config->limited_color_range = true; + } else { + if (pipe_config->has_hdmi_sink && + intel_sdvo->color_range == HDMI_COLOR_RANGE_16_235) + pipe_config->limited_color_range = true; } - if (intel_sdvo->color_range) - pipe_config->limited_color_range = true; - /* Clock computation needs to happen after pixel multiplier. */ if (intel_sdvo->is_tv) i9xx_adjust_sdvo_tv_clock(pipe_config); @@ -1174,7 +1175,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, return true; } -static void intel_sdvo_mode_set(struct intel_encoder *intel_encoder) +static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder) { struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -1223,7 +1224,7 @@ static void intel_sdvo_mode_set(struct intel_encoder *intel_encoder) if (!intel_sdvo_set_target_input(intel_sdvo)) return; - if (intel_sdvo->has_hdmi_monitor) { + if (crtc->config.has_hdmi_sink) { intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI); intel_sdvo_set_colorimetry(intel_sdvo, SDVO_COLORIMETRY_RGB256); @@ -1258,8 +1259,8 @@ static void intel_sdvo_mode_set(struct intel_encoder *intel_encoder) /* The real mode polarity is set by the SDVO commands, using * struct intel_sdvo_dtd. */ sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH; - if (!HAS_PCH_SPLIT(dev) && intel_sdvo->is_hdmi) - sdvox |= intel_sdvo->color_range; + if (!HAS_PCH_SPLIT(dev) && crtc->config.limited_color_range) + sdvox |= HDMI_COLOR_RANGE_16_235; if (INTEL_INFO(dev)->gen < 5) sdvox |= SDVO_BORDER_ENABLE; } else { @@ -1349,6 +1350,8 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder, u8 val; bool ret; + sdvox = I915_READ(intel_sdvo->sdvo_reg); + ret = intel_sdvo_get_input_timing(intel_sdvo, &dtd); if (!ret) { /* Some sdvo encoders are not spec compliant and don't @@ -1377,13 +1380,14 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder, * other platfroms. */ if (IS_I915G(dev) || IS_I915GM(dev)) { - sdvox = I915_READ(intel_sdvo->sdvo_reg); pipe_config->pixel_multiplier = ((sdvox & SDVO_PORT_MULTIPLY_MASK) >> SDVO_PORT_MULTIPLY_SHIFT) + 1; } - dotclock = pipe_config->port_clock / pipe_config->pixel_multiplier; + dotclock = pipe_config->port_clock; + if (pipe_config->pixel_multiplier) + dotclock /= pipe_config->pixel_multiplier; if (HAS_PCH_SPLIT(dev)) ironlake_check_encoder_dotclock(pipe_config, dotclock); @@ -1406,6 +1410,15 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder, } } + if (sdvox & HDMI_COLOR_RANGE_16_235) + pipe_config->limited_color_range = true; + + if (intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ENCODE, + &val, 1)) { + if (val == SDVO_ENCODE_HDMI) + pipe_config->has_hdmi_sink = true; + } + WARN(encoder_pixel_multiplier != pipe_config->pixel_multiplier, "SDVO pixel multiplier mismatch, port: %i, encoder: %i\n", pipe_config->pixel_multiplier, encoder_pixel_multiplier); @@ -1461,7 +1474,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder) u32 temp; bool input1, input2; int i; - u8 status; + bool success; temp = I915_READ(intel_sdvo->sdvo_reg); if ((temp & SDVO_ENABLE) == 0) { @@ -1475,12 +1488,12 @@ static void intel_enable_sdvo(struct intel_encoder *encoder) for (i = 0; i < 2; i++) intel_wait_for_vblank(dev, intel_crtc->pipe); - status = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2); + success = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2); /* Warn if the device reported failure to sync. * A lot of SDVO devices fail to notify of sync, but it's * a given it the status is a success, we succeeded. */ - if (status == SDVO_CMD_STATUS_SUCCESS && !input1) { + if (success && !input1) { DRM_DEBUG_KMS("First %s output reported failure to " "sync\n", SDVO_NAME(intel_sdvo)); } @@ -1732,7 +1745,7 @@ intel_sdvo_detect(struct drm_connector *connector, bool force) enum drm_connector_status ret; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ATTACHED_DISPLAYS, @@ -1794,7 +1807,7 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector) struct edid *edid; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); /* set the bus switch and get the modes */ edid = intel_sdvo_get_edid(connector); @@ -1892,7 +1905,7 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector) int i; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); /* Read the list of supported input resolutions for the selected TV * format. @@ -1929,7 +1942,7 @@ static void intel_sdvo_get_lvds_modes(struct drm_connector *connector) struct drm_display_mode *newmode; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, drm_get_connector_name(connector)); + connector->base.id, connector->name); /* * Fetch modes from VBT. For SDVO prefer the VBT mode since some @@ -2382,24 +2395,62 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo) } static void +intel_sdvo_connector_unregister(struct intel_connector *intel_connector) +{ + struct drm_connector *drm_connector; + struct intel_sdvo *sdvo_encoder; + + drm_connector = &intel_connector->base; + sdvo_encoder = intel_attached_sdvo(&intel_connector->base); + + sysfs_remove_link(&drm_connector->kdev->kobj, + sdvo_encoder->ddc.dev.kobj.name); + intel_connector_unregister(intel_connector); +} + +static int intel_sdvo_connector_init(struct intel_sdvo_connector *connector, struct intel_sdvo *encoder) { - drm_connector_init(encoder->base.base.dev, - &connector->base.base, + struct drm_connector *drm_connector; + int ret; + + drm_connector = &connector->base.base; + ret = drm_connector_init(encoder->base.base.dev, + drm_connector, &intel_sdvo_connector_funcs, connector->base.base.connector_type); + if (ret < 0) + return ret; - drm_connector_helper_add(&connector->base.base, + drm_connector_helper_add(drm_connector, &intel_sdvo_connector_helper_funcs); connector->base.base.interlace_allowed = 1; connector->base.base.doublescan_allowed = 0; connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB; connector->base.get_hw_state = intel_sdvo_connector_get_hw_state; + connector->base.unregister = intel_sdvo_connector_unregister; intel_connector_attach_encoder(&connector->base, &encoder->base); - drm_sysfs_connector_add(&connector->base.base); + ret = drm_sysfs_connector_add(drm_connector); + if (ret < 0) + goto err1; + + ret = sysfs_create_link(&drm_connector->kdev->kobj, + &encoder->ddc.dev.kobj, + encoder->ddc.dev.kobj.name); + if (ret < 0) + goto err2; + + return 0; + +err2: + drm_sysfs_connector_remove(drm_connector); +err1: + drm_connector_cleanup(drm_connector); + + return ret; } static void @@ -2459,7 +2510,11 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device) intel_sdvo->is_hdmi = true; } - intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo); + if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { + kfree(intel_sdvo_connector); + return false; + } + if (intel_sdvo->is_hdmi) intel_sdvo_add_hdmi_properties(intel_sdvo, intel_sdvo_connector); @@ -2490,7 +2545,10 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type) intel_sdvo->is_tv = true; - intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo); + if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { + kfree(intel_sdvo_connector); + return false; + } if (!intel_sdvo_tv_create_property(intel_sdvo, intel_sdvo_connector, type)) goto err; @@ -2534,8 +2592,11 @@ intel_sdvo_analog_init(struct intel_sdvo *intel_sdvo, int device) intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB1; } - intel_sdvo_connector_init(intel_sdvo_connector, - intel_sdvo); + if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { + kfree(intel_sdvo_connector); + return false; + } + return true; } @@ -2566,7 +2627,11 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS1; } - intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo); + if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { + kfree(intel_sdvo_connector); + return false; + } + if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector)) goto err; @@ -2947,7 +3012,7 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob) intel_encoder->compute_config = intel_sdvo_compute_config; intel_encoder->disable = intel_disable_sdvo; - intel_encoder->mode_set = intel_sdvo_mode_set; + intel_encoder->pre_enable = intel_sdvo_pre_enable; intel_encoder->enable = intel_enable_sdvo; intel_encoder->get_hw_state = intel_sdvo_get_hw_state; intel_encoder->get_config = intel_sdvo_get_config; @@ -2980,7 +3045,7 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob) * simplistic anyway to express such constraints, so just give up on * cloning for SDVO encoders. */ - intel_sdvo->base.cloneable = false; + intel_sdvo->base.cloneable = 0; intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg); diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c index 0954f132726..01d841ea314 100644 --- a/drivers/gpu/drm/i915/intel_sideband.c +++ b/drivers/gpu/drm/i915/intel_sideband.c @@ -29,12 +29,21 @@ * IOSF sideband, see VLV2_SidebandMsg_HAS.docx and * VLV_VLV2_PUNIT_HAS_0.8.docx */ + +/* Standard MMIO read, non-posted */ +#define SB_MRD_NP 0x00 +/* Standard MMIO write, non-posted */ +#define SB_MWR_NP 0x01 +/* Private register read, double-word addressing, non-posted */ +#define SB_CRRDDA_NP 0x06 +/* Private register write, double-word addressing, non-posted */ +#define SB_CRWRDA_NP 0x07 + static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn, u32 port, u32 opcode, u32 addr, u32 *val) { u32 cmd, be = 0xf, bar = 0; - bool is_read = (opcode == PUNIT_OPCODE_REG_READ || - opcode == DPIO_OPCODE_REG_READ); + bool is_read = (opcode == SB_MRD_NP || opcode == SB_CRRDDA_NP); cmd = (devfn << IOSF_DEVFN_SHIFT) | (opcode << IOSF_OPCODE_SHIFT) | (port << IOSF_PORT_SHIFT) | (be << IOSF_BYTE_ENABLES_SHIFT) | @@ -74,7 +83,7 @@ u32 vlv_punit_read(struct drm_i915_private *dev_priv, u8 addr) mutex_lock(&dev_priv->dpio_lock); vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_PUNIT, - PUNIT_OPCODE_REG_READ, addr, &val); + SB_CRRDDA_NP, addr, &val); mutex_unlock(&dev_priv->dpio_lock); return val; @@ -86,7 +95,7 @@ void vlv_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val) mutex_lock(&dev_priv->dpio_lock); vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_PUNIT, - PUNIT_OPCODE_REG_WRITE, addr, &val); + SB_CRWRDA_NP, addr, &val); mutex_unlock(&dev_priv->dpio_lock); } @@ -95,7 +104,7 @@ u32 vlv_bunit_read(struct drm_i915_private *dev_priv, u32 reg) u32 val = 0; vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_BUNIT, - PUNIT_OPCODE_REG_READ, reg, &val); + SB_CRRDDA_NP, reg, &val); return val; } @@ -103,7 +112,7 @@ u32 vlv_bunit_read(struct drm_i915_private *dev_priv, u32 reg) void vlv_bunit_write(struct drm_i915_private *dev_priv, u32 reg, u32 val) { vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_BUNIT, - PUNIT_OPCODE_REG_WRITE, reg, &val); + SB_CRWRDA_NP, reg, &val); } u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr) @@ -114,7 +123,7 @@ u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr) mutex_lock(&dev_priv->dpio_lock); vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_NC, - PUNIT_OPCODE_REG_READ, addr, &val); + SB_CRRDDA_NP, addr, &val); mutex_unlock(&dev_priv->dpio_lock); return val; @@ -124,56 +133,56 @@ u32 vlv_gpio_nc_read(struct drm_i915_private *dev_priv, u32 reg) { u32 val = 0; vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_GPIO_NC, - PUNIT_OPCODE_REG_READ, reg, &val); + SB_CRRDDA_NP, reg, &val); return val; } void vlv_gpio_nc_write(struct drm_i915_private *dev_priv, u32 reg, u32 val) { vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_GPIO_NC, - PUNIT_OPCODE_REG_WRITE, reg, &val); + SB_CRWRDA_NP, reg, &val); } u32 vlv_cck_read(struct drm_i915_private *dev_priv, u32 reg) { u32 val = 0; vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_CCK, - PUNIT_OPCODE_REG_READ, reg, &val); + SB_CRRDDA_NP, reg, &val); return val; } void vlv_cck_write(struct drm_i915_private *dev_priv, u32 reg, u32 val) { vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_CCK, - PUNIT_OPCODE_REG_WRITE, reg, &val); + SB_CRWRDA_NP, reg, &val); } u32 vlv_ccu_read(struct drm_i915_private *dev_priv, u32 reg) { u32 val = 0; vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_CCU, - PUNIT_OPCODE_REG_READ, reg, &val); + SB_CRRDDA_NP, reg, &val); return val; } void vlv_ccu_write(struct drm_i915_private *dev_priv, u32 reg, u32 val) { vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_CCU, - PUNIT_OPCODE_REG_WRITE, reg, &val); + SB_CRWRDA_NP, reg, &val); } u32 vlv_gps_core_read(struct drm_i915_private *dev_priv, u32 reg) { u32 val = 0; vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_GPS_CORE, - PUNIT_OPCODE_REG_READ, reg, &val); + SB_CRRDDA_NP, reg, &val); return val; } void vlv_gps_core_write(struct drm_i915_private *dev_priv, u32 reg, u32 val) { vlv_sideband_rw(dev_priv, PCI_DEVFN(2, 0), IOSF_PORT_GPS_CORE, - PUNIT_OPCODE_REG_WRITE, reg, &val); + SB_CRWRDA_NP, reg, &val); } u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum pipe pipe, int reg) @@ -181,14 +190,22 @@ u32 vlv_dpio_read(struct drm_i915_private *dev_priv, enum pipe pipe, int reg) u32 val = 0; vlv_sideband_rw(dev_priv, DPIO_DEVFN, DPIO_PHY_IOSF_PORT(DPIO_PHY(pipe)), - DPIO_OPCODE_REG_READ, reg, &val); + SB_MRD_NP, reg, &val); + + /* + * FIXME: There might be some registers where all 1's is a valid value, + * so ideally we should check the register offset instead... + */ + WARN(val == 0xffffffff, "DPIO read pipe %c reg 0x%x == 0x%x\n", + pipe_name(pipe), reg, val); + return val; } void vlv_dpio_write(struct drm_i915_private *dev_priv, enum pipe pipe, int reg, u32 val) { vlv_sideband_rw(dev_priv, DPIO_DEVFN, DPIO_PHY_IOSF_PORT(DPIO_PHY(pipe)), - DPIO_OPCODE_REG_WRITE, reg, &val); + SB_MWR_NP, reg, &val); } /* SBI access */ @@ -253,13 +270,13 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg) { u32 val = 0; - vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_FLISDSI, - DPIO_OPCODE_REG_READ, reg, &val); + vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRRDDA_NP, + reg, &val); return val; } void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val) { - vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_FLISDSI, - DPIO_OPCODE_REG_WRITE, reg, &val); + vlv_sideband_rw(dev_priv, DPIO_DEVFN, IOSF_PORT_FLISDSI, SB_CRWRDA_NP, + reg, &val); } diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index 716a3c9c075..9a17b4e92ef 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -37,6 +37,106 @@ #include <drm/i915_drm.h> #include "i915_drv.h" +static int usecs_to_scanlines(const struct drm_display_mode *mode, int usecs) +{ + /* paranoia */ + if (!mode->crtc_htotal) + return 1; + + return DIV_ROUND_UP(usecs * mode->crtc_clock, 1000 * mode->crtc_htotal); +} + +static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl_count) +{ + struct drm_device *dev = crtc->base.dev; + const struct drm_display_mode *mode = &crtc->config.adjusted_mode; + enum pipe pipe = crtc->pipe; + long timeout = msecs_to_jiffies_timeout(1); + int scanline, min, max, vblank_start; + DEFINE_WAIT(wait); + + WARN_ON(!drm_modeset_is_locked(&crtc->base.mutex)); + + vblank_start = mode->crtc_vblank_start; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + vblank_start = DIV_ROUND_UP(vblank_start, 2); + + /* FIXME needs to be calibrated sensibly */ + min = vblank_start - usecs_to_scanlines(mode, 100); + max = vblank_start - 1; + + if (min <= 0 || max <= 0) + return false; + + if (WARN_ON(drm_vblank_get(dev, pipe))) + return false; + + local_irq_disable(); + + trace_i915_pipe_update_start(crtc, min, max); + + for (;;) { + /* + * prepare_to_wait() has a memory barrier, which guarantees + * other CPUs can see the task state update by the time we + * read the scanline. + */ + prepare_to_wait(&crtc->vbl_wait, &wait, TASK_UNINTERRUPTIBLE); + + scanline = intel_get_crtc_scanline(crtc); + if (scanline < min || scanline > max) + break; + + if (timeout <= 0) { + DRM_ERROR("Potential atomic update failure on pipe %c\n", + pipe_name(crtc->pipe)); + break; + } + + local_irq_enable(); + + timeout = schedule_timeout(timeout); + + local_irq_disable(); + } + + finish_wait(&crtc->vbl_wait, &wait); + + drm_vblank_put(dev, pipe); + + *start_vbl_count = dev->driver->get_vblank_counter(dev, pipe); + + trace_i915_pipe_update_vblank_evaded(crtc, min, max, *start_vbl_count); + + return true; +} + +static void intel_pipe_update_end(struct intel_crtc *crtc, u32 start_vbl_count) +{ + struct drm_device *dev = crtc->base.dev; + enum pipe pipe = crtc->pipe; + u32 end_vbl_count = dev->driver->get_vblank_counter(dev, pipe); + + trace_i915_pipe_update_end(crtc, end_vbl_count); + + local_irq_enable(); + + if (start_vbl_count != end_vbl_count) + DRM_ERROR("Atomic update failure on pipe %c (start=%u end=%u)\n", + pipe_name(pipe), start_vbl_count, end_vbl_count); +} + +static void intel_update_primary_plane(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + int reg = DSPCNTR(crtc->plane); + + if (crtc->primary_enabled) + I915_WRITE(reg, I915_READ(reg) | DISPLAY_PLANE_ENABLE); + else + I915_WRITE(reg, I915_READ(reg) & ~DISPLAY_PLANE_ENABLE); +} + static void vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, struct drm_framebuffer *fb, @@ -48,11 +148,14 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, struct drm_device *dev = dplane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(dplane); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_plane->pipe; int plane = intel_plane->plane; u32 sprctl; unsigned long sprsurf_offset, linear_offset; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); + u32 start_vbl_count; + bool atomic_update; sprctl = I915_READ(SPCNTR(pipe, plane)); @@ -124,9 +227,6 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, crtc_w--; crtc_h--; - I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]); - I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x); - linear_offset = y * fb->pitches[0] + x * pixel_size; sprsurf_offset = intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, @@ -134,6 +234,13 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, fb->pitches[0]); linear_offset -= sprsurf_offset; + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + + intel_update_primary_plane(intel_crtc); + + I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]); + I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x); + if (obj->tiling_mode != I915_TILING_NONE) I915_WRITE(SPTILEOFF(pipe, plane), (y << 16) | x); else @@ -143,7 +250,11 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc, I915_WRITE(SPCNTR(pipe, plane), sprctl); I915_WRITE(SPSURF(pipe, plane), i915_gem_obj_ggtt_offset(obj) + sprsurf_offset); - POSTING_READ(SPSURF(pipe, plane)); + + intel_flush_primary_plane(dev_priv, intel_crtc->plane); + + if (atomic_update) + intel_pipe_update_end(intel_crtc, start_vbl_count); } static void @@ -152,14 +263,25 @@ vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) struct drm_device *dev = dplane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(dplane); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_plane->pipe; int plane = intel_plane->plane; + u32 start_vbl_count; + bool atomic_update; + + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + + intel_update_primary_plane(intel_crtc); I915_WRITE(SPCNTR(pipe, plane), I915_READ(SPCNTR(pipe, plane)) & ~SP_ENABLE); /* Activate double buffered register update */ I915_WRITE(SPSURF(pipe, plane), 0); - POSTING_READ(SPSURF(pipe, plane)); + + intel_flush_primary_plane(dev_priv, intel_crtc->plane); + + if (atomic_update) + intel_pipe_update_end(intel_crtc, start_vbl_count); intel_update_sprite_watermarks(dplane, crtc, 0, 0, false, false); } @@ -226,10 +348,13 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(plane); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_plane->pipe; u32 sprctl, sprscale = 0; unsigned long sprsurf_offset, linear_offset; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); + u32 start_vbl_count; + bool atomic_update; sprctl = I915_READ(SPRCTL(pipe)); @@ -293,15 +418,19 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, if (crtc_w != src_w || crtc_h != src_h) sprscale = SPRITE_SCALE_ENABLE | (src_w << 16) | src_h; - I915_WRITE(SPRSTRIDE(pipe), fb->pitches[0]); - I915_WRITE(SPRPOS(pipe), (crtc_y << 16) | crtc_x); - linear_offset = y * fb->pitches[0] + x * pixel_size; sprsurf_offset = intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); linear_offset -= sprsurf_offset; + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + + intel_update_primary_plane(intel_crtc); + + I915_WRITE(SPRSTRIDE(pipe), fb->pitches[0]); + I915_WRITE(SPRPOS(pipe), (crtc_y << 16) | crtc_x); + /* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET * register */ if (IS_HASWELL(dev) || IS_BROADWELL(dev)) @@ -317,7 +446,11 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, I915_WRITE(SPRCTL(pipe), sprctl); I915_WRITE(SPRSURF(pipe), i915_gem_obj_ggtt_offset(obj) + sprsurf_offset); - POSTING_READ(SPRSURF(pipe)); + + intel_flush_primary_plane(dev_priv, intel_crtc->plane); + + if (atomic_update) + intel_pipe_update_end(intel_crtc, start_vbl_count); } static void @@ -326,7 +459,14 @@ ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(plane); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_plane->pipe; + u32 start_vbl_count; + bool atomic_update; + + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + + intel_update_primary_plane(intel_crtc); I915_WRITE(SPRCTL(pipe), I915_READ(SPRCTL(pipe)) & ~SPRITE_ENABLE); /* Can't leave the scaler enabled... */ @@ -334,7 +474,11 @@ ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) I915_WRITE(SPRSCALE(pipe), 0); /* Activate double buffered register update */ I915_WRITE(SPRSURF(pipe), 0); - POSTING_READ(SPRSURF(pipe)); + + intel_flush_primary_plane(dev_priv, intel_crtc->plane); + + if (atomic_update) + intel_pipe_update_end(intel_crtc, start_vbl_count); /* * Avoid underruns when disabling the sprite. @@ -410,10 +554,13 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(plane); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_plane->pipe; unsigned long dvssurf_offset, linear_offset; u32 dvscntr, dvsscale; int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0); + u32 start_vbl_count; + bool atomic_update; dvscntr = I915_READ(DVSCNTR(pipe)); @@ -472,15 +619,19 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, if (crtc_w != src_w || crtc_h != src_h) dvsscale = DVS_SCALE_ENABLE | (src_w << 16) | src_h; - I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]); - I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x); - linear_offset = y * fb->pitches[0] + x * pixel_size; dvssurf_offset = intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode, pixel_size, fb->pitches[0]); linear_offset -= dvssurf_offset; + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + + intel_update_primary_plane(intel_crtc); + + I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]); + I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x); + if (obj->tiling_mode != I915_TILING_NONE) I915_WRITE(DVSTILEOFF(pipe), (y << 16) | x); else @@ -491,7 +642,11 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, I915_WRITE(DVSCNTR(pipe), dvscntr); I915_WRITE(DVSSURF(pipe), i915_gem_obj_ggtt_offset(obj) + dvssurf_offset); - POSTING_READ(DVSSURF(pipe)); + + intel_flush_primary_plane(dev_priv, intel_crtc->plane); + + if (atomic_update) + intel_pipe_update_end(intel_crtc, start_vbl_count); } static void @@ -500,14 +655,25 @@ ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_plane *intel_plane = to_intel_plane(plane); + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_plane->pipe; + u32 start_vbl_count; + bool atomic_update; + + atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count); + + intel_update_primary_plane(intel_crtc); I915_WRITE(DVSCNTR(pipe), I915_READ(DVSCNTR(pipe)) & ~DVS_ENABLE); /* Disable the scaler */ I915_WRITE(DVSSCALE(pipe), 0); /* Flush double buffered register updates */ I915_WRITE(DVSSURF(pipe), 0); - POSTING_READ(DVSSURF(pipe)); + + intel_flush_primary_plane(dev_priv, intel_crtc->plane); + + if (atomic_update) + intel_pipe_update_end(intel_crtc, start_vbl_count); /* * Avoid underruns when disabling the sprite. @@ -519,20 +685,18 @@ ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) } static void -intel_enable_primary(struct drm_crtc *crtc) +intel_post_enable_primary(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int reg = DSPCNTR(intel_crtc->plane); - if (intel_crtc->primary_enabled) - return; - - intel_crtc->primary_enabled = true; - - I915_WRITE(reg, I915_READ(reg) | DISPLAY_PLANE_ENABLE); - intel_flush_primary_plane(dev_priv, intel_crtc->plane); + /* + * BDW signals flip done immediately if the plane + * is disabled, even if the plane enable is already + * armed to occur at the next vblank :( + */ + if (IS_BROADWELL(dev)) + intel_wait_for_vblank(dev, intel_crtc->pipe); /* * FIXME IPS should be fine as long as one plane is @@ -540,10 +704,7 @@ intel_enable_primary(struct drm_crtc *crtc) * when going from primary only to sprite only and vice * versa. */ - if (intel_crtc->config.ips_enabled) { - intel_wait_for_vblank(dev, intel_crtc->pipe); - hsw_enable_ips(intel_crtc); - } + hsw_enable_ips(intel_crtc); mutex_lock(&dev->struct_mutex); intel_update_fbc(dev); @@ -551,17 +712,11 @@ intel_enable_primary(struct drm_crtc *crtc) } static void -intel_disable_primary(struct drm_crtc *crtc) +intel_pre_disable_primary(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - int reg = DSPCNTR(intel_crtc->plane); - - if (!intel_crtc->primary_enabled) - return; - - intel_crtc->primary_enabled = false; mutex_lock(&dev->struct_mutex); if (dev_priv->fbc.plane == intel_crtc->plane) @@ -575,9 +730,6 @@ intel_disable_primary(struct drm_crtc *crtc) * versa. */ hsw_disable_ips(intel_crtc); - - I915_WRITE(reg, I915_READ(reg) & ~DISPLAY_PLANE_ENABLE); - intel_flush_primary_plane(dev_priv, intel_crtc->plane); } static int @@ -671,7 +823,7 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, struct drm_i915_gem_object *obj = intel_fb->obj; struct drm_i915_gem_object *old_obj = intel_plane->obj; int ret; - bool disable_primary = false; + bool primary_enabled; bool visible; int hscale, vscale; int max_scale, min_scale; @@ -842,8 +994,8 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, * If the sprite is completely covering the primary plane, * we can disable the primary and save power. */ - disable_primary = drm_rect_equals(&dst, &clip) && !colorkey_enabled(intel_plane); - WARN_ON(disable_primary && !visible && intel_crtc->active); + primary_enabled = !drm_rect_equals(&dst, &clip) || colorkey_enabled(intel_plane); + WARN_ON(!primary_enabled && !visible && intel_crtc->active); mutex_lock(&dev->struct_mutex); @@ -870,12 +1022,15 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, intel_plane->obj = obj; if (intel_crtc->active) { - /* - * Be sure to re-enable the primary before the sprite is no longer - * covering it fully. - */ - if (!disable_primary) - intel_enable_primary(crtc); + bool primary_was_enabled = intel_crtc->primary_enabled; + + intel_crtc->primary_enabled = primary_enabled; + + if (primary_was_enabled != primary_enabled) + intel_crtc_wait_for_pending_flips(crtc); + + if (primary_was_enabled && !primary_enabled) + intel_pre_disable_primary(crtc); if (visible) intel_plane->update_plane(plane, crtc, fb, obj, @@ -884,8 +1039,8 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc, else intel_plane->disable_plane(plane, crtc); - if (disable_primary) - intel_disable_primary(crtc); + if (!primary_was_enabled && primary_enabled) + intel_post_enable_primary(crtc); } /* Unpin old obj after new one is active to avoid ugliness */ @@ -923,8 +1078,14 @@ intel_disable_plane(struct drm_plane *plane) intel_crtc = to_intel_crtc(plane->crtc); if (intel_crtc->active) { - intel_enable_primary(plane->crtc); + bool primary_was_enabled = intel_crtc->primary_enabled; + + intel_crtc->primary_enabled = true; + intel_plane->disable_plane(plane, plane->crtc); + + if (!primary_was_enabled && intel_crtc->primary_enabled) + intel_post_enable_primary(plane->crtc); } if (intel_plane->obj) { diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 22cf0f4ba24..67c6c9a2eb1 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -934,7 +934,86 @@ intel_tv_compute_config(struct intel_encoder *encoder, return true; } -static void intel_tv_mode_set(struct intel_encoder *encoder) +static void +set_tv_mode_timings(struct drm_i915_private *dev_priv, + const struct tv_mode *tv_mode, + bool burst_ena) +{ + u32 hctl1, hctl2, hctl3; + u32 vctl1, vctl2, vctl3, vctl4, vctl5, vctl6, vctl7; + + hctl1 = (tv_mode->hsync_end << TV_HSYNC_END_SHIFT) | + (tv_mode->htotal << TV_HTOTAL_SHIFT); + + hctl2 = (tv_mode->hburst_start << 16) | + (tv_mode->hburst_len << TV_HBURST_LEN_SHIFT); + + if (burst_ena) + hctl2 |= TV_BURST_ENA; + + hctl3 = (tv_mode->hblank_start << TV_HBLANK_START_SHIFT) | + (tv_mode->hblank_end << TV_HBLANK_END_SHIFT); + + vctl1 = (tv_mode->nbr_end << TV_NBR_END_SHIFT) | + (tv_mode->vi_end_f1 << TV_VI_END_F1_SHIFT) | + (tv_mode->vi_end_f2 << TV_VI_END_F2_SHIFT); + + vctl2 = (tv_mode->vsync_len << TV_VSYNC_LEN_SHIFT) | + (tv_mode->vsync_start_f1 << TV_VSYNC_START_F1_SHIFT) | + (tv_mode->vsync_start_f2 << TV_VSYNC_START_F2_SHIFT); + + vctl3 = (tv_mode->veq_len << TV_VEQ_LEN_SHIFT) | + (tv_mode->veq_start_f1 << TV_VEQ_START_F1_SHIFT) | + (tv_mode->veq_start_f2 << TV_VEQ_START_F2_SHIFT); + + if (tv_mode->veq_ena) + vctl3 |= TV_EQUAL_ENA; + + vctl4 = (tv_mode->vburst_start_f1 << TV_VBURST_START_F1_SHIFT) | + (tv_mode->vburst_end_f1 << TV_VBURST_END_F1_SHIFT); + + vctl5 = (tv_mode->vburst_start_f2 << TV_VBURST_START_F2_SHIFT) | + (tv_mode->vburst_end_f2 << TV_VBURST_END_F2_SHIFT); + + vctl6 = (tv_mode->vburst_start_f3 << TV_VBURST_START_F3_SHIFT) | + (tv_mode->vburst_end_f3 << TV_VBURST_END_F3_SHIFT); + + vctl7 = (tv_mode->vburst_start_f4 << TV_VBURST_START_F4_SHIFT) | + (tv_mode->vburst_end_f4 << TV_VBURST_END_F4_SHIFT); + + I915_WRITE(TV_H_CTL_1, hctl1); + I915_WRITE(TV_H_CTL_2, hctl2); + I915_WRITE(TV_H_CTL_3, hctl3); + I915_WRITE(TV_V_CTL_1, vctl1); + I915_WRITE(TV_V_CTL_2, vctl2); + I915_WRITE(TV_V_CTL_3, vctl3); + I915_WRITE(TV_V_CTL_4, vctl4); + I915_WRITE(TV_V_CTL_5, vctl5); + I915_WRITE(TV_V_CTL_6, vctl6); + I915_WRITE(TV_V_CTL_7, vctl7); +} + +static void set_color_conversion(struct drm_i915_private *dev_priv, + const struct color_conversion *color_conversion) +{ + if (!color_conversion) + return; + + I915_WRITE(TV_CSC_Y, (color_conversion->ry << 16) | + color_conversion->gy); + I915_WRITE(TV_CSC_Y2, (color_conversion->by << 16) | + color_conversion->ay); + I915_WRITE(TV_CSC_U, (color_conversion->ru << 16) | + color_conversion->gu); + I915_WRITE(TV_CSC_U2, (color_conversion->bu << 16) | + color_conversion->au); + I915_WRITE(TV_CSC_V, (color_conversion->rv << 16) | + color_conversion->gv); + I915_WRITE(TV_CSC_V2, (color_conversion->bv << 16) | + color_conversion->av); +} + +static void intel_tv_pre_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; @@ -942,14 +1021,13 @@ static void intel_tv_mode_set(struct intel_encoder *encoder) struct intel_tv *intel_tv = enc_to_tv(encoder); const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv); u32 tv_ctl; - u32 hctl1, hctl2, hctl3; - u32 vctl1, vctl2, vctl3, vctl4, vctl5, vctl6, vctl7; u32 scctl1, scctl2, scctl3; int i, j; const struct video_levels *video_levels; const struct color_conversion *color_conversion; bool burst_ena; - int pipe = intel_crtc->pipe; + int xpos = 0x0, ypos = 0x0; + unsigned int xsize, ysize; if (!tv_mode) return; /* can't happen (mode_prepare prevents this) */ @@ -982,44 +1060,6 @@ static void intel_tv_mode_set(struct intel_encoder *encoder) burst_ena = tv_mode->burst_ena; break; } - hctl1 = (tv_mode->hsync_end << TV_HSYNC_END_SHIFT) | - (tv_mode->htotal << TV_HTOTAL_SHIFT); - - hctl2 = (tv_mode->hburst_start << 16) | - (tv_mode->hburst_len << TV_HBURST_LEN_SHIFT); - - if (burst_ena) - hctl2 |= TV_BURST_ENA; - - hctl3 = (tv_mode->hblank_start << TV_HBLANK_START_SHIFT) | - (tv_mode->hblank_end << TV_HBLANK_END_SHIFT); - - vctl1 = (tv_mode->nbr_end << TV_NBR_END_SHIFT) | - (tv_mode->vi_end_f1 << TV_VI_END_F1_SHIFT) | - (tv_mode->vi_end_f2 << TV_VI_END_F2_SHIFT); - - vctl2 = (tv_mode->vsync_len << TV_VSYNC_LEN_SHIFT) | - (tv_mode->vsync_start_f1 << TV_VSYNC_START_F1_SHIFT) | - (tv_mode->vsync_start_f2 << TV_VSYNC_START_F2_SHIFT); - - vctl3 = (tv_mode->veq_len << TV_VEQ_LEN_SHIFT) | - (tv_mode->veq_start_f1 << TV_VEQ_START_F1_SHIFT) | - (tv_mode->veq_start_f2 << TV_VEQ_START_F2_SHIFT); - - if (tv_mode->veq_ena) - vctl3 |= TV_EQUAL_ENA; - - vctl4 = (tv_mode->vburst_start_f1 << TV_VBURST_START_F1_SHIFT) | - (tv_mode->vburst_end_f1 << TV_VBURST_END_F1_SHIFT); - - vctl5 = (tv_mode->vburst_start_f2 << TV_VBURST_START_F2_SHIFT) | - (tv_mode->vburst_end_f2 << TV_VBURST_END_F2_SHIFT); - - vctl6 = (tv_mode->vburst_start_f3 << TV_VBURST_START_F3_SHIFT) | - (tv_mode->vburst_end_f3 << TV_VBURST_END_F3_SHIFT); - - vctl7 = (tv_mode->vburst_start_f4 << TV_VBURST_START_F4_SHIFT) | - (tv_mode->vburst_end_f4 << TV_VBURST_END_F4_SHIFT); if (intel_crtc->pipe == 1) tv_ctl |= TV_ENC_PIPEB_SELECT; @@ -1051,37 +1091,16 @@ static void intel_tv_mode_set(struct intel_encoder *encoder) tv_mode->dda3_inc << TV_SCDDA3_INC_SHIFT; /* Enable two fixes for the chips that need them. */ - if (dev->pdev->device < 0x2772) + if (IS_I915GM(dev)) tv_ctl |= TV_ENC_C0_FIX | TV_ENC_SDP_FIX; - I915_WRITE(TV_H_CTL_1, hctl1); - I915_WRITE(TV_H_CTL_2, hctl2); - I915_WRITE(TV_H_CTL_3, hctl3); - I915_WRITE(TV_V_CTL_1, vctl1); - I915_WRITE(TV_V_CTL_2, vctl2); - I915_WRITE(TV_V_CTL_3, vctl3); - I915_WRITE(TV_V_CTL_4, vctl4); - I915_WRITE(TV_V_CTL_5, vctl5); - I915_WRITE(TV_V_CTL_6, vctl6); - I915_WRITE(TV_V_CTL_7, vctl7); + set_tv_mode_timings(dev_priv, tv_mode, burst_ena); + I915_WRITE(TV_SC_CTL_1, scctl1); I915_WRITE(TV_SC_CTL_2, scctl2); I915_WRITE(TV_SC_CTL_3, scctl3); - if (color_conversion) { - I915_WRITE(TV_CSC_Y, (color_conversion->ry << 16) | - color_conversion->gy); - I915_WRITE(TV_CSC_Y2, (color_conversion->by << 16) | - color_conversion->ay); - I915_WRITE(TV_CSC_U, (color_conversion->ru << 16) | - color_conversion->gu); - I915_WRITE(TV_CSC_U2, (color_conversion->bu << 16) | - color_conversion->au); - I915_WRITE(TV_CSC_V, (color_conversion->rv << 16) | - color_conversion->gv); - I915_WRITE(TV_CSC_V2, (color_conversion->bv << 16) | - color_conversion->av); - } + set_color_conversion(dev_priv, color_conversion); if (INTEL_INFO(dev)->gen >= 4) I915_WRITE(TV_CLR_KNOBS, 0x00404000); @@ -1092,46 +1111,25 @@ static void intel_tv_mode_set(struct intel_encoder *encoder) I915_WRITE(TV_CLR_LEVEL, ((video_levels->black << TV_BLACK_LEVEL_SHIFT) | (video_levels->blank << TV_BLANK_LEVEL_SHIFT))); - { - int pipeconf_reg = PIPECONF(pipe); - int dspcntr_reg = DSPCNTR(intel_crtc->plane); - int pipeconf = I915_READ(pipeconf_reg); - int dspcntr = I915_READ(dspcntr_reg); - int xpos = 0x0, ypos = 0x0; - unsigned int xsize, ysize; - /* Pipe must be off here */ - I915_WRITE(dspcntr_reg, dspcntr & ~DISPLAY_PLANE_ENABLE); - intel_flush_primary_plane(dev_priv, intel_crtc->plane); - - /* Wait for vblank for the disable to take effect */ - if (IS_GEN2(dev)) - intel_wait_for_vblank(dev, intel_crtc->pipe); - - I915_WRITE(pipeconf_reg, pipeconf & ~PIPECONF_ENABLE); - /* Wait for vblank for the disable to take effect. */ - intel_wait_for_pipe_off(dev, intel_crtc->pipe); - - /* Filter ctl must be set before TV_WIN_SIZE */ - I915_WRITE(TV_FILTER_CTL_1, TV_AUTO_SCALE); - xsize = tv_mode->hblank_start - tv_mode->hblank_end; - if (tv_mode->progressive) - ysize = tv_mode->nbr_end + 1; - else - ysize = 2*tv_mode->nbr_end + 1; - - xpos += intel_tv->margin[TV_MARGIN_LEFT]; - ypos += intel_tv->margin[TV_MARGIN_TOP]; - xsize -= (intel_tv->margin[TV_MARGIN_LEFT] + - intel_tv->margin[TV_MARGIN_RIGHT]); - ysize -= (intel_tv->margin[TV_MARGIN_TOP] + - intel_tv->margin[TV_MARGIN_BOTTOM]); - I915_WRITE(TV_WIN_POS, (xpos<<16)|ypos); - I915_WRITE(TV_WIN_SIZE, (xsize<<16)|ysize); - - I915_WRITE(pipeconf_reg, pipeconf); - I915_WRITE(dspcntr_reg, dspcntr); - intel_flush_primary_plane(dev_priv, intel_crtc->plane); - } + + assert_pipe_disabled(dev_priv, intel_crtc->pipe); + + /* Filter ctl must be set before TV_WIN_SIZE */ + I915_WRITE(TV_FILTER_CTL_1, TV_AUTO_SCALE); + xsize = tv_mode->hblank_start - tv_mode->hblank_end; + if (tv_mode->progressive) + ysize = tv_mode->nbr_end + 1; + else + ysize = 2*tv_mode->nbr_end + 1; + + xpos += intel_tv->margin[TV_MARGIN_LEFT]; + ypos += intel_tv->margin[TV_MARGIN_TOP]; + xsize -= (intel_tv->margin[TV_MARGIN_LEFT] + + intel_tv->margin[TV_MARGIN_RIGHT]); + ysize -= (intel_tv->margin[TV_MARGIN_TOP] + + intel_tv->margin[TV_MARGIN_BOTTOM]); + I915_WRITE(TV_WIN_POS, (xpos<<16)|ypos); + I915_WRITE(TV_WIN_SIZE, (xsize<<16)|ysize); j = 0; for (i = 0; i < 60; i++) @@ -1189,8 +1187,8 @@ intel_tv_detect_type(struct intel_tv *intel_tv, if (connector->polled & DRM_CONNECTOR_POLL_HPD) { spin_lock_irqsave(&dev_priv->irq_lock, irqflags); i915_disable_pipestat(dev_priv, 0, - PIPE_HOTPLUG_INTERRUPT_ENABLE | - PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); + PIPE_HOTPLUG_INTERRUPT_STATUS | + PIPE_HOTPLUG_TV_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } @@ -1266,8 +1264,8 @@ intel_tv_detect_type(struct intel_tv *intel_tv, if (connector->polled & DRM_CONNECTOR_POLL_HPD) { spin_lock_irqsave(&dev_priv->irq_lock, irqflags); i915_enable_pipestat(dev_priv, 0, - PIPE_HOTPLUG_INTERRUPT_ENABLE | - PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); + PIPE_HOTPLUG_INTERRUPT_STATUS | + PIPE_HOTPLUG_TV_INTERRUPT_STATUS); spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } @@ -1316,17 +1314,18 @@ intel_tv_detect(struct drm_connector *connector, bool force) int type; DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force=%d\n", - connector->base.id, drm_get_connector_name(connector), + connector->base.id, connector->name, force); mode = reported_modes[0]; if (force) { struct intel_load_detect_pipe tmp; + struct drm_modeset_acquire_ctx ctx; - if (intel_get_load_detect_pipe(connector, &mode, &tmp)) { + if (intel_get_load_detect_pipe(connector, &mode, &tmp, &ctx)) { type = intel_tv_detect_type(intel_tv, connector); - intel_release_load_detect_pipe(connector, &tmp); + intel_release_load_detect_pipe(connector, &tmp, &ctx); } else return connector_status_unknown; } else @@ -1536,9 +1535,14 @@ static int tv_is_present_in_vbt(struct drm_device *dev) /* * If the device type is not TV, continue. */ - if (p_child->old.device_type != DEVICE_TYPE_INT_TV && - p_child->old.device_type != DEVICE_TYPE_TV) + switch (p_child->old.device_type) { + case DEVICE_TYPE_INT_TV: + case DEVICE_TYPE_TV: + case DEVICE_TYPE_TV_SVIDEO_COMPOSITE: + break; + default: continue; + } /* Only when the addin_offset is non-zero, it is regarded * as present. */ @@ -1629,18 +1633,18 @@ intel_tv_init(struct drm_device *dev) intel_encoder->compute_config = intel_tv_compute_config; intel_encoder->get_config = intel_tv_get_config; - intel_encoder->mode_set = intel_tv_mode_set; + intel_encoder->pre_enable = intel_tv_pre_enable; intel_encoder->enable = intel_enable_tv; intel_encoder->disable = intel_disable_tv; intel_encoder->get_hw_state = intel_tv_get_hw_state; intel_connector->get_hw_state = intel_connector_get_hw_state; + intel_connector->unregister = intel_connector_unregister; intel_connector_attach_encoder(intel_connector, intel_encoder); intel_encoder->type = INTEL_OUTPUT_TVOUT; intel_encoder->crtc_mask = (1 << 0) | (1 << 1); - intel_encoder->cloneable = false; + intel_encoder->cloneable = 0; intel_encoder->base.possible_crtcs = ((1 << 0) | (1 << 1)); - intel_encoder->base.possible_clones = (1 << INTEL_OUTPUT_TVOUT); intel_tv->type = DRM_MODE_CONNECTOR_Unknown; /* BIOS margin values */ diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 87df68f5f50..4f6fef7ac06 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -40,6 +40,12 @@ #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__) +static void +assert_device_not_suspended(struct drm_i915_private *dev_priv) +{ + WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended, + "Device suspended\n"); +} static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv) { @@ -83,14 +89,14 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, __gen6_gt_wait_for_thread_c0(dev_priv); } -static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) +static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff)); /* something from same cacheline, but !FORCEWAKE_MT */ __raw_posting_read(dev_priv, ECOBUS); } -static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, +static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, int fw_engine) { u32 forcewake_ack; @@ -136,14 +142,16 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, gen6_gt_check_fifodbg(dev_priv); } -static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, +static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, int fw_engine) { __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); /* something from same cacheline, but !FORCEWAKE_MT */ __raw_posting_read(dev_priv, ECOBUS); - gen6_gt_check_fifodbg(dev_priv); + + if (IS_GEN7(dev_priv->dev)) + gen6_gt_check_fifodbg(dev_priv); } static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) @@ -177,6 +185,8 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv) { __raw_i915_write32(dev_priv, FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff)); + __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, + _MASKED_BIT_DISABLE(0xffff)); /* something from same cacheline, but !FORCEWAKE_VLV */ __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV); } @@ -245,73 +255,115 @@ static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, } -void vlv_force_wake_get(struct drm_i915_private *dev_priv, - int fw_engine) +static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) { unsigned long irqflags; spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - if (FORCEWAKE_RENDER & fw_engine) { - if (dev_priv->uncore.fw_rendercount++ == 0) - dev_priv->uncore.funcs.force_wake_get(dev_priv, - FORCEWAKE_RENDER); - } - if (FORCEWAKE_MEDIA & fw_engine) { - if (dev_priv->uncore.fw_mediacount++ == 0) - dev_priv->uncore.funcs.force_wake_get(dev_priv, - FORCEWAKE_MEDIA); - } + + if (fw_engine & FORCEWAKE_RENDER && + dev_priv->uncore.fw_rendercount++ != 0) + fw_engine &= ~FORCEWAKE_RENDER; + if (fw_engine & FORCEWAKE_MEDIA && + dev_priv->uncore.fw_mediacount++ != 0) + fw_engine &= ~FORCEWAKE_MEDIA; + + if (fw_engine) + dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } -void vlv_force_wake_put(struct drm_i915_private *dev_priv, - int fw_engine) +static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) { unsigned long irqflags; spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); - if (FORCEWAKE_RENDER & fw_engine) { - WARN_ON(dev_priv->uncore.fw_rendercount == 0); - if (--dev_priv->uncore.fw_rendercount == 0) - dev_priv->uncore.funcs.force_wake_put(dev_priv, - FORCEWAKE_RENDER); + if (fw_engine & FORCEWAKE_RENDER) { + WARN_ON(!dev_priv->uncore.fw_rendercount); + if (--dev_priv->uncore.fw_rendercount != 0) + fw_engine &= ~FORCEWAKE_RENDER; } - if (FORCEWAKE_MEDIA & fw_engine) { - WARN_ON(dev_priv->uncore.fw_mediacount == 0); - if (--dev_priv->uncore.fw_mediacount == 0) - dev_priv->uncore.funcs.force_wake_put(dev_priv, - FORCEWAKE_MEDIA); + if (fw_engine & FORCEWAKE_MEDIA) { + WARN_ON(!dev_priv->uncore.fw_mediacount); + if (--dev_priv->uncore.fw_mediacount != 0) + fw_engine &= ~FORCEWAKE_MEDIA; } + if (fw_engine) + dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } -static void gen6_force_wake_work(struct work_struct *work) +static void gen6_force_wake_timer(unsigned long arg) { - struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), uncore.force_wake_work.work); + struct drm_i915_private *dev_priv = (void *)arg; unsigned long irqflags; + assert_device_not_suspended(dev_priv); + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + WARN_ON(!dev_priv->uncore.forcewake_count); + if (--dev_priv->uncore.forcewake_count == 0) dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + + intel_runtime_pm_put(dev_priv); } -static void intel_uncore_forcewake_reset(struct drm_device *dev) +static void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) { struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long irqflags; - if (IS_VALLEYVIEW(dev)) { + if (del_timer_sync(&dev_priv->uncore.force_wake_timer)) + gen6_force_wake_timer((unsigned long)dev_priv); + + /* Hold uncore.lock across reset to prevent any register access + * with forcewake not set correctly + */ + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + + if (IS_VALLEYVIEW(dev)) vlv_force_wake_reset(dev_priv); - } else if (INTEL_INFO(dev)->gen >= 6) { + else if (IS_GEN6(dev) || IS_GEN7(dev)) __gen6_gt_force_wake_reset(dev_priv); - if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) - __gen6_gt_force_wake_mt_reset(dev_priv); + + if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_GEN8(dev)) + __gen7_gt_force_wake_mt_reset(dev_priv); + + if (restore) { /* If reset with a user forcewake, try to restore */ + unsigned fw = 0; + + if (IS_VALLEYVIEW(dev)) { + if (dev_priv->uncore.fw_rendercount) + fw |= FORCEWAKE_RENDER; + + if (dev_priv->uncore.fw_mediacount) + fw |= FORCEWAKE_MEDIA; + } else { + if (dev_priv->uncore.forcewake_count) + fw = FORCEWAKE_ALL; + } + + if (fw) + dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); + + if (IS_GEN6(dev) || IS_GEN7(dev)) + dev_priv->uncore.fifo_count = + __raw_i915_read32(dev_priv, GTFIFOCTL) & + GT_FIFO_FREE_ENTRIES_MASK; + } else { + dev_priv->uncore.forcewake_count = 0; + dev_priv->uncore.fw_rendercount = 0; + dev_priv->uncore.fw_mediacount = 0; } + + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } void intel_uncore_early_sanitize(struct drm_device *dev) @@ -321,7 +373,7 @@ void intel_uncore_early_sanitize(struct drm_device *dev) if (HAS_FPGA_DBG_UNCLAIMED(dev)) __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); - if (IS_HASWELL(dev) && + if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) { /* The docs do not explain exactly how the calculation can be * made. It is somewhat guessable, but for now, it's always @@ -337,29 +389,13 @@ void intel_uncore_early_sanitize(struct drm_device *dev) __raw_i915_write32(dev_priv, GTFIFODBG, __raw_i915_read32(dev_priv, GTFIFODBG)); - intel_uncore_forcewake_reset(dev); + intel_uncore_forcewake_reset(dev, false); } void intel_uncore_sanitize(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - u32 reg_val; - /* BIOS often leaves RC6 enabled, but disable it for hw init */ intel_disable_gt_powersave(dev); - - /* Turn off power gate, require especially for the BIOS less system */ - if (IS_VALLEYVIEW(dev)) { - - mutex_lock(&dev_priv->rps.hw_lock); - reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS); - - if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT)) - vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0); - - mutex_unlock(&dev_priv->rps.hw_lock); - - } } /* @@ -393,31 +429,57 @@ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) { unsigned long irqflags; + bool delayed = false; if (!dev_priv->uncore.funcs.force_wake_put) return; /* Redirect to VLV specific routine */ - if (IS_VALLEYVIEW(dev_priv->dev)) - return vlv_force_wake_put(dev_priv, fw_engine); + if (IS_VALLEYVIEW(dev_priv->dev)) { + vlv_force_wake_put(dev_priv, fw_engine); + goto out; + } spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + WARN_ON(!dev_priv->uncore.forcewake_count); + if (--dev_priv->uncore.forcewake_count == 0) { dev_priv->uncore.forcewake_count++; - mod_delayed_work(dev_priv->wq, - &dev_priv->uncore.force_wake_work, - 1); + delayed = true; + mod_timer_pinned(&dev_priv->uncore.force_wake_timer, + jiffies + 1); } spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); - intel_runtime_pm_put(dev_priv); +out: + if (!delayed) + intel_runtime_pm_put(dev_priv); +} + +void assert_force_wake_inactive(struct drm_i915_private *dev_priv) +{ + if (!dev_priv->uncore.funcs.force_wake_get) + return; + + WARN_ON(dev_priv->uncore.forcewake_count > 0); } /* We give fast paths for the really cool registers */ #define NEEDS_FORCE_WAKE(dev_priv, reg) \ ((reg) < 0x40000 && (reg) != FORCEWAKE) +#define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \ + (((reg) >= 0x2000 && (reg) < 0x4000) ||\ + ((reg) >= 0x5000 && (reg) < 0x8000) ||\ + ((reg) >= 0xB000 && (reg) < 0x12000) ||\ + ((reg) >= 0x2E000 && (reg) < 0x30000)) + +#define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)\ + (((reg) >= 0x12000 && (reg) < 0x14000) ||\ + ((reg) >= 0x22000 && (reg) < 0x24000) ||\ + ((reg) >= 0x30000 && (reg) < 0x40000)) + static void ilk_dummy_write(struct drm_i915_private *dev_priv) { @@ -446,16 +508,10 @@ hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg) } } -static void -assert_device_not_suspended(struct drm_i915_private *dev_priv) -{ - WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended, - "Device suspended\n"); -} - #define REG_READ_HEADER(x) \ unsigned long irqflags; \ u##x val = 0; \ + assert_device_not_suspended(dev_priv); \ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) #define REG_READ_FOOTER \ @@ -484,14 +540,13 @@ gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ static u##x \ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ REG_READ_HEADER(x); \ - if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ - if (dev_priv->uncore.forcewake_count == 0) \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, \ - FORCEWAKE_ALL); \ + if (dev_priv->uncore.forcewake_count == 0 && \ + NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ + dev_priv->uncore.funcs.force_wake_get(dev_priv, \ + FORCEWAKE_ALL); \ val = __raw_i915_read##x(dev_priv, reg); \ - if (dev_priv->uncore.forcewake_count == 0) \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, \ - FORCEWAKE_ALL); \ + dev_priv->uncore.funcs.force_wake_put(dev_priv, \ + FORCEWAKE_ALL); \ } else { \ val = __raw_i915_read##x(dev_priv, reg); \ } \ @@ -502,27 +557,19 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ static u##x \ vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ unsigned fwengine = 0; \ - unsigned *fwcount; \ REG_READ_HEADER(x); \ - if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ - fwengine = FORCEWAKE_RENDER; \ - fwcount = &dev_priv->uncore.fw_rendercount; \ - } \ - else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ - fwengine = FORCEWAKE_MEDIA; \ - fwcount = &dev_priv->uncore.fw_mediacount; \ + if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ + if (dev_priv->uncore.fw_rendercount == 0) \ + fwengine = FORCEWAKE_RENDER; \ + } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ + if (dev_priv->uncore.fw_mediacount == 0) \ + fwengine = FORCEWAKE_MEDIA; \ } \ - if (fwengine != 0) { \ - if ((*fwcount)++ == 0) \ - (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \ - fwengine); \ - val = __raw_i915_read##x(dev_priv, reg); \ - if (--(*fwcount) == 0) \ - (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \ - fwengine); \ - } else { \ - val = __raw_i915_read##x(dev_priv, reg); \ - } \ + if (fwengine) \ + dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \ + val = __raw_i915_read##x(dev_priv, reg); \ + if (fwengine) \ + dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \ REG_READ_FOOTER; \ } @@ -554,6 +601,7 @@ __gen4_read(64) #define REG_WRITE_HEADER \ unsigned long irqflags; \ trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ + assert_device_not_suspended(dev_priv); \ spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) #define REG_WRITE_FOOTER \ @@ -584,7 +632,6 @@ gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ } \ - assert_device_not_suspended(dev_priv); \ __raw_i915_write##x(dev_priv, reg, val); \ if (unlikely(__fifo_ret)) { \ gen6_gt_check_fifodbg(dev_priv); \ @@ -600,7 +647,6 @@ hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ } \ - assert_device_not_suspended(dev_priv); \ hsw_unclaimed_reg_clear(dev_priv, reg); \ __raw_i915_write##x(dev_priv, reg, val); \ if (unlikely(__fifo_ret)) { \ @@ -634,16 +680,17 @@ static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg) #define __gen8_write(x) \ static void \ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ - bool __needs_put = reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg); \ REG_WRITE_HEADER; \ - if (__needs_put) { \ - dev_priv->uncore.funcs.force_wake_get(dev_priv, \ - FORCEWAKE_ALL); \ - } \ - __raw_i915_write##x(dev_priv, reg, val); \ - if (__needs_put) { \ - dev_priv->uncore.funcs.force_wake_put(dev_priv, \ - FORCEWAKE_ALL); \ + if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \ + if (dev_priv->uncore.forcewake_count == 0) \ + dev_priv->uncore.funcs.force_wake_get(dev_priv, \ + FORCEWAKE_ALL); \ + __raw_i915_write##x(dev_priv, reg, val); \ + if (dev_priv->uncore.forcewake_count == 0) \ + dev_priv->uncore.funcs.force_wake_put(dev_priv, \ + FORCEWAKE_ALL); \ + } else { \ + __raw_i915_write##x(dev_priv, reg, val); \ } \ REG_WRITE_FOOTER; \ } @@ -681,15 +728,17 @@ void intel_uncore_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work, - gen6_force_wake_work); + setup_timer(&dev_priv->uncore.force_wake_timer, + gen6_force_wake_timer, (unsigned long)dev_priv); + + intel_uncore_early_sanitize(dev); if (IS_VALLEYVIEW(dev)) { dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; } else if (IS_HASWELL(dev) || IS_GEN8(dev)) { - dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get; - dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put; + dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get; + dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put; } else if (IS_IVYBRIDGE(dev)) { u32 ecobus; @@ -703,16 +752,16 @@ void intel_uncore_init(struct drm_device *dev) * forcewake being disabled. */ mutex_lock(&dev->struct_mutex); - __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL); + __gen7_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL); ecobus = __raw_i915_read32(dev_priv, ECOBUS); - __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL); + __gen7_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL); mutex_unlock(&dev->struct_mutex); if (ecobus & FORCEWAKE_MT_ENABLE) { dev_priv->uncore.funcs.force_wake_get = - __gen6_gt_force_wake_mt_get; + __gen7_gt_force_wake_mt_get; dev_priv->uncore.funcs.force_wake_put = - __gen6_gt_force_wake_mt_put; + __gen7_gt_force_wake_mt_put; } else { DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); DRM_INFO("when using vblank-synced partial screen updates.\n"); @@ -792,20 +841,20 @@ void intel_uncore_init(struct drm_device *dev) void intel_uncore_fini(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - - flush_delayed_work(&dev_priv->uncore.force_wake_work); - /* Paranoia: make sure we have disabled everything before we exit. */ intel_uncore_sanitize(dev); + intel_uncore_forcewake_reset(dev, false); } +#define GEN_RANGE(l, h) GENMASK(h, l) + static const struct register_whitelist { uint64_t offset; uint32_t size; - uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */ + /* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */ + uint32_t gen_bitmask; } whitelist[] = { - { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0x1F0 }, + { RING_TIMESTAMP(RENDER_RING_BASE), 8, GEN_RANGE(4, 8) }, }; int i915_reg_read_ioctl(struct drm_device *dev, @@ -814,7 +863,7 @@ int i915_reg_read_ioctl(struct drm_device *dev, struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_reg_read *reg = data; struct register_whitelist const *entry = whitelist; - int i; + int i, ret = 0; for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { if (entry->offset == reg->offset && @@ -825,6 +874,8 @@ int i915_reg_read_ioctl(struct drm_device *dev, if (i == ARRAY_SIZE(whitelist)) return -EINVAL; + intel_runtime_pm_get(dev_priv); + switch (entry->size) { case 8: reg->val = I915_READ64(reg->offset); @@ -840,10 +891,13 @@ int i915_reg_read_ioctl(struct drm_device *dev, break; default: WARN_ON(1); - return -EINVAL; + ret = -EINVAL; + goto out; } - return 0; +out: + intel_runtime_pm_put(dev_priv); + return ret; } int i915_get_reset_stats_ioctl(struct drm_device *dev, @@ -852,6 +906,7 @@ int i915_get_reset_stats_ioctl(struct drm_device *dev, struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_reset_stats *args = data; struct i915_ctx_hang_stats *hs; + struct intel_context *ctx; int ret; if (args->flags || args->pad) @@ -864,11 +919,12 @@ int i915_get_reset_stats_ioctl(struct drm_device *dev, if (ret) return ret; - hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id); - if (IS_ERR(hs)) { + ctx = i915_gem_context_get(file->driver_priv, args->ctx_id); + if (IS_ERR(ctx)) { mutex_unlock(&dev->struct_mutex); - return PTR_ERR(hs); + return PTR_ERR(ctx); } + hs = &ctx->hang_stats; if (capable(CAP_SYS_ADMIN)) args->reset_count = i915_reset_count(&dev_priv->gpu_error); @@ -894,6 +950,9 @@ static int i965_do_reset(struct drm_device *dev) { int ret; + /* FIXME: i965g/gm need a display save/restore for gpu reset. */ + return -ENODEV; + /* * Set the domains we want to reset (GRDOM/bits 2 and 3) as * well as the reset bit (GR/bit 0). Setting the GR bit @@ -905,7 +964,6 @@ static int i965_do_reset(struct drm_device *dev) if (ret) return ret; - /* We can't reset render&media without also resetting display ... */ pci_write_config_byte(dev->pdev, I965_GDRST, GRDOM_MEDIA | GRDOM_RESET_ENABLE); @@ -918,38 +976,64 @@ static int i965_do_reset(struct drm_device *dev) return 0; } +static int g4x_do_reset(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + pci_write_config_byte(dev->pdev, I965_GDRST, + GRDOM_RENDER | GRDOM_RESET_ENABLE); + ret = wait_for(i965_reset_complete(dev), 500); + if (ret) + return ret; + + /* WaVcpClkGateDisableForMediaReset:ctg,elk */ + I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); + POSTING_READ(VDECCLK_GATE_D); + + pci_write_config_byte(dev->pdev, I965_GDRST, + GRDOM_MEDIA | GRDOM_RESET_ENABLE); + ret = wait_for(i965_reset_complete(dev), 500); + if (ret) + return ret; + + /* WaVcpClkGateDisableForMediaReset:ctg,elk */ + I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); + POSTING_READ(VDECCLK_GATE_D); + + pci_write_config_byte(dev->pdev, I965_GDRST, 0); + + return 0; +} + static int ironlake_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 gdrst; int ret; - gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); - gdrst &= ~GRDOM_MASK; I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, - gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE); - ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); + ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); + ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & + ILK_GRDOM_RESET_ENABLE) == 0, 500); if (ret) return ret; - /* We can't reset render&media without also resetting display ... */ - gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); - gdrst &= ~GRDOM_MASK; I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, - gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE); - return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); + ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); + ret = wait_for((I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & + ILK_GRDOM_RESET_ENABLE) == 0, 500); + if (ret) + return ret; + + I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 0); + + return 0; } static int gen6_do_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; - unsigned long irqflags; - - /* Hold uncore.lock across reset to prevent any register access - * with forcewake not set correctly - */ - spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); /* Reset the chip */ @@ -962,18 +1046,8 @@ static int gen6_do_reset(struct drm_device *dev) /* Spin waiting for the device to ack the reset request */ ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500); - intel_uncore_forcewake_reset(dev); + intel_uncore_forcewake_reset(dev, true); - /* If reset with a user forcewake, try to restore, otherwise turn it off */ - if (dev_priv->uncore.forcewake_count) - dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); - else - dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); - - /* Restore fifo count */ - dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; - - spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); return ret; } @@ -984,7 +1058,11 @@ int intel_gpu_reset(struct drm_device *dev) case 7: case 6: return gen6_do_reset(dev); case 5: return ironlake_do_reset(dev); - case 4: return i965_do_reset(dev); + case 4: + if (IS_G4X(dev)) + return g4x_do_reset(dev); + else + return i965_do_reset(dev); default: return -ENODEV; } } |