source: src/linux/universal/linux-4.4/drivers/gpu/drm/i915/i915_irq.c @ 29197

Last change on this file since 29197 was 29197, checked in by BrainSlayer, 17 months ago

update kernel

File size: 126.9 KB
Line 
1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
2 */
3/*
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 */
28
29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31#include <linux/sysrq.h>
32#include <linux/slab.h>
33#include <linux/circ_buf.h>
34#include <drm/drmP.h>
35#include <drm/i915_drm.h>
36#include "i915_drv.h"
37#include "i915_trace.h"
38#include "intel_drv.h"
39
40/**
41 * DOC: interrupt handling
42 *
43 * These functions provide the basic support for enabling and disabling the
44 * interrupt handling support. There's a lot more functionality in i915_irq.c
45 * and related files, but that will be described in separate chapters.
46 */
47
48static const u32 hpd_ilk[HPD_NUM_PINS] = {
49        [HPD_PORT_A] = DE_DP_A_HOTPLUG,
50};
51
52static const u32 hpd_ivb[HPD_NUM_PINS] = {
53        [HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
54};
55
56static const u32 hpd_bdw[HPD_NUM_PINS] = {
57        [HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
58};
59
60static const u32 hpd_ibx[HPD_NUM_PINS] = {
61        [HPD_CRT] = SDE_CRT_HOTPLUG,
62        [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
63        [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
64        [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
65        [HPD_PORT_D] = SDE_PORTD_HOTPLUG
66};
67
68static const u32 hpd_cpt[HPD_NUM_PINS] = {
69        [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70        [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71        [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
72        [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
73        [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
74};
75
76static const u32 hpd_spt[HPD_NUM_PINS] = {
77        [HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
78        [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
79        [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
80        [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
81        [HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
82};
83
84static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85        [HPD_CRT] = CRT_HOTPLUG_INT_EN,
86        [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
87        [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
88        [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
89        [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
90        [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
91};
92
93static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94        [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
95        [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
96        [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
97        [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
98        [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
99        [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
100};
101
102static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103        [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
104        [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
105        [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
106        [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
107        [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
108        [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
109};
110
111/* BXT hpd list */
112static const u32 hpd_bxt[HPD_NUM_PINS] = {
113        [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114        [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
115        [HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
116};
117
118/* IIR can theoretically queue up two events. Be paranoid. */
119#define GEN8_IRQ_RESET_NDX(type, which) do { \
120        I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
121        POSTING_READ(GEN8_##type##_IMR(which)); \
122        I915_WRITE(GEN8_##type##_IER(which), 0); \
123        I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
124        POSTING_READ(GEN8_##type##_IIR(which)); \
125        I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
126        POSTING_READ(GEN8_##type##_IIR(which)); \
127} while (0)
128
129#define GEN5_IRQ_RESET(type) do { \
130        I915_WRITE(type##IMR, 0xffffffff); \
131        POSTING_READ(type##IMR); \
132        I915_WRITE(type##IER, 0); \
133        I915_WRITE(type##IIR, 0xffffffff); \
134        POSTING_READ(type##IIR); \
135        I915_WRITE(type##IIR, 0xffffffff); \
136        POSTING_READ(type##IIR); \
137} while (0)
138
139/*
140 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
141 */
142static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, u32 reg)
143{
144        u32 val = I915_READ(reg);
145
146        if (val == 0)
147                return;
148
149        WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
150             reg, val);
151        I915_WRITE(reg, 0xffffffff);
152        POSTING_READ(reg);
153        I915_WRITE(reg, 0xffffffff);
154        POSTING_READ(reg);
155}
156
157#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
158        gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
159        I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
160        I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
161        POSTING_READ(GEN8_##type##_IMR(which)); \
162} while (0)
163
164#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
165        gen5_assert_iir_is_zero(dev_priv, type##IIR); \
166        I915_WRITE(type##IER, (ier_val)); \
167        I915_WRITE(type##IMR, (imr_val)); \
168        POSTING_READ(type##IMR); \
169} while (0)
170
171static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
172
173/* For display hotplug interrupt */
174static inline void
175i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
176                                     uint32_t mask,
177                                     uint32_t bits)
178{
179        uint32_t val;
180
181        assert_spin_locked(&dev_priv->irq_lock);
182        WARN_ON(bits & ~mask);
183
184        val = I915_READ(PORT_HOTPLUG_EN);
185        val &= ~mask;
186        val |= bits;
187        I915_WRITE(PORT_HOTPLUG_EN, val);
188}
189
190/**
191 * i915_hotplug_interrupt_update - update hotplug interrupt enable
192 * @dev_priv: driver private
193 * @mask: bits to update
194 * @bits: bits to enable
195 * NOTE: the HPD enable bits are modified both inside and outside
196 * of an interrupt context. To avoid that read-modify-write cycles
197 * interfer, these bits are protected by a spinlock. Since this
198 * function is usually not called from a context where the lock is
199 * held already, this function acquires the lock itself. A non-locking
200 * version is also available.
201 */
202void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
203                                   uint32_t mask,
204                                   uint32_t bits)
205{
206        spin_lock_irq(&dev_priv->irq_lock);
207        i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
208        spin_unlock_irq(&dev_priv->irq_lock);
209}
210
211/**
212 * ilk_update_display_irq - update DEIMR
213 * @dev_priv: driver private
214 * @interrupt_mask: mask of interrupt bits to update
215 * @enabled_irq_mask: mask of interrupt bits to enable
216 */
217static void ilk_update_display_irq(struct drm_i915_private *dev_priv,
218                                   uint32_t interrupt_mask,
219                                   uint32_t enabled_irq_mask)
220{
221        uint32_t new_val;
222
223        assert_spin_locked(&dev_priv->irq_lock);
224
225        WARN_ON(enabled_irq_mask & ~interrupt_mask);
226
227        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
228                return;
229
230        new_val = dev_priv->irq_mask;
231        new_val &= ~interrupt_mask;
232        new_val |= (~enabled_irq_mask & interrupt_mask);
233
234        if (new_val != dev_priv->irq_mask) {
235                dev_priv->irq_mask = new_val;
236                I915_WRITE(DEIMR, dev_priv->irq_mask);
237                POSTING_READ(DEIMR);
238        }
239}
240
241void
242ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
243{
244        ilk_update_display_irq(dev_priv, mask, mask);
245}
246
247void
248ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
249{
250        ilk_update_display_irq(dev_priv, mask, 0);
251}
252
253/**
254 * ilk_update_gt_irq - update GTIMR
255 * @dev_priv: driver private
256 * @interrupt_mask: mask of interrupt bits to update
257 * @enabled_irq_mask: mask of interrupt bits to enable
258 */
259static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
260                              uint32_t interrupt_mask,
261                              uint32_t enabled_irq_mask)
262{
263        assert_spin_locked(&dev_priv->irq_lock);
264
265        WARN_ON(enabled_irq_mask & ~interrupt_mask);
266
267        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
268                return;
269
270        dev_priv->gt_irq_mask &= ~interrupt_mask;
271        dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
272        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
273        POSTING_READ(GTIMR);
274}
275
276void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
277{
278        ilk_update_gt_irq(dev_priv, mask, mask);
279}
280
281void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
282{
283        ilk_update_gt_irq(dev_priv, mask, 0);
284}
285
286static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
287{
288        return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
289}
290
291static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
292{
293        return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
294}
295
296static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
297{
298        return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
299}
300
301/**
302  * snb_update_pm_irq - update GEN6_PMIMR
303  * @dev_priv: driver private
304  * @interrupt_mask: mask of interrupt bits to update
305  * @enabled_irq_mask: mask of interrupt bits to enable
306  */
307static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
308                              uint32_t interrupt_mask,
309                              uint32_t enabled_irq_mask)
310{
311        uint32_t new_val;
312
313        WARN_ON(enabled_irq_mask & ~interrupt_mask);
314
315        assert_spin_locked(&dev_priv->irq_lock);
316
317        new_val = dev_priv->pm_irq_mask;
318        new_val &= ~interrupt_mask;
319        new_val |= (~enabled_irq_mask & interrupt_mask);
320
321        if (new_val != dev_priv->pm_irq_mask) {
322                dev_priv->pm_irq_mask = new_val;
323                I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
324                POSTING_READ(gen6_pm_imr(dev_priv));
325        }
326}
327
328void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
329{
330        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
331                return;
332
333        snb_update_pm_irq(dev_priv, mask, mask);
334}
335
336static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
337                                  uint32_t mask)
338{
339        snb_update_pm_irq(dev_priv, mask, 0);
340}
341
342void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
343{
344        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
345                return;
346
347        __gen6_disable_pm_irq(dev_priv, mask);
348}
349
350void gen6_reset_rps_interrupts(struct drm_device *dev)
351{
352        struct drm_i915_private *dev_priv = dev->dev_private;
353        uint32_t reg = gen6_pm_iir(dev_priv);
354
355        spin_lock_irq(&dev_priv->irq_lock);
356        I915_WRITE(reg, dev_priv->pm_rps_events);
357        I915_WRITE(reg, dev_priv->pm_rps_events);
358        POSTING_READ(reg);
359        dev_priv->rps.pm_iir = 0;
360        spin_unlock_irq(&dev_priv->irq_lock);
361}
362
363void gen6_enable_rps_interrupts(struct drm_device *dev)
364{
365        struct drm_i915_private *dev_priv = dev->dev_private;
366
367        spin_lock_irq(&dev_priv->irq_lock);
368
369        WARN_ON(dev_priv->rps.pm_iir);
370        WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
371        dev_priv->rps.interrupts_enabled = true;
372        I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
373                                dev_priv->pm_rps_events);
374        gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
375
376        spin_unlock_irq(&dev_priv->irq_lock);
377}
378
379u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
380{
381        /*
382         * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
383         * if GEN6_PM_UP_EI_EXPIRED is masked.
384         *
385         * TODO: verify if this can be reproduced on VLV,CHV.
386         */
387        if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
388                mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
389
390        if (INTEL_INFO(dev_priv)->gen >= 8)
391                mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
392
393        return mask;
394}
395
396void gen6_disable_rps_interrupts(struct drm_device *dev)
397{
398        struct drm_i915_private *dev_priv = dev->dev_private;
399
400        spin_lock_irq(&dev_priv->irq_lock);
401        dev_priv->rps.interrupts_enabled = false;
402        spin_unlock_irq(&dev_priv->irq_lock);
403
404        cancel_work_sync(&dev_priv->rps.work);
405
406        spin_lock_irq(&dev_priv->irq_lock);
407
408        I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
409
410        __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
411        I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
412                                ~dev_priv->pm_rps_events);
413
414        spin_unlock_irq(&dev_priv->irq_lock);
415
416        synchronize_irq(dev->irq);
417}
418
419/**
420  * bdw_update_port_irq - update DE port interrupt
421  * @dev_priv: driver private
422  * @interrupt_mask: mask of interrupt bits to update
423  * @enabled_irq_mask: mask of interrupt bits to enable
424  */
425static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
426                                uint32_t interrupt_mask,
427                                uint32_t enabled_irq_mask)
428{
429        uint32_t new_val;
430        uint32_t old_val;
431
432        assert_spin_locked(&dev_priv->irq_lock);
433
434        WARN_ON(enabled_irq_mask & ~interrupt_mask);
435
436        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
437                return;
438
439        old_val = I915_READ(GEN8_DE_PORT_IMR);
440
441        new_val = old_val;
442        new_val &= ~interrupt_mask;
443        new_val |= (~enabled_irq_mask & interrupt_mask);
444
445        if (new_val != old_val) {
446                I915_WRITE(GEN8_DE_PORT_IMR, new_val);
447                POSTING_READ(GEN8_DE_PORT_IMR);
448        }
449}
450
451/**
452 * ibx_display_interrupt_update - update SDEIMR
453 * @dev_priv: driver private
454 * @interrupt_mask: mask of interrupt bits to update
455 * @enabled_irq_mask: mask of interrupt bits to enable
456 */
457void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
458                                  uint32_t interrupt_mask,
459                                  uint32_t enabled_irq_mask)
460{
461        uint32_t sdeimr = I915_READ(SDEIMR);
462        sdeimr &= ~interrupt_mask;
463        sdeimr |= (~enabled_irq_mask & interrupt_mask);
464
465        WARN_ON(enabled_irq_mask & ~interrupt_mask);
466
467        assert_spin_locked(&dev_priv->irq_lock);
468
469        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
470                return;
471
472        I915_WRITE(SDEIMR, sdeimr);
473        POSTING_READ(SDEIMR);
474}
475
476static void
477__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
478                       u32 enable_mask, u32 status_mask)
479{
480        u32 reg = PIPESTAT(pipe);
481        u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
482
483        assert_spin_locked(&dev_priv->irq_lock);
484        WARN_ON(!intel_irqs_enabled(dev_priv));
485
486        if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
487                      status_mask & ~PIPESTAT_INT_STATUS_MASK,
488                      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
489                      pipe_name(pipe), enable_mask, status_mask))
490                return;
491
492        if ((pipestat & enable_mask) == enable_mask)
493                return;
494
495        dev_priv->pipestat_irq_mask[pipe] |= status_mask;
496
497        /* Enable the interrupt, clear any pending status */
498        pipestat |= enable_mask | status_mask;
499        I915_WRITE(reg, pipestat);
500        POSTING_READ(reg);
501}
502
503static void
504__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
505                        u32 enable_mask, u32 status_mask)
506{
507        u32 reg = PIPESTAT(pipe);
508        u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
509
510        assert_spin_locked(&dev_priv->irq_lock);
511        WARN_ON(!intel_irqs_enabled(dev_priv));
512
513        if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
514                      status_mask & ~PIPESTAT_INT_STATUS_MASK,
515                      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
516                      pipe_name(pipe), enable_mask, status_mask))
517                return;
518
519        if ((pipestat & enable_mask) == 0)
520                return;
521
522        dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
523
524        pipestat &= ~enable_mask;
525        I915_WRITE(reg, pipestat);
526        POSTING_READ(reg);
527}
528
529static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
530{
531        u32 enable_mask = status_mask << 16;
532
533        /*
534         * On pipe A we don't support the PSR interrupt yet,
535         * on pipe B and C the same bit MBZ.
536         */
537        if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
538                return 0;
539        /*
540         * On pipe B and C we don't support the PSR interrupt yet, on pipe
541         * A the same bit is for perf counters which we don't use either.
542         */
543        if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
544                return 0;
545
546        enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
547                         SPRITE0_FLIP_DONE_INT_EN_VLV |
548                         SPRITE1_FLIP_DONE_INT_EN_VLV);
549        if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
550                enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
551        if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
552                enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
553
554        return enable_mask;
555}
556
557void
558i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
559                     u32 status_mask)
560{
561        u32 enable_mask;
562
563        if (IS_VALLEYVIEW(dev_priv->dev))
564                enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
565                                                           status_mask);
566        else
567                enable_mask = status_mask << 16;
568        __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
569}
570
571void
572i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
573                      u32 status_mask)
574{
575        u32 enable_mask;
576
577        if (IS_VALLEYVIEW(dev_priv->dev))
578                enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
579                                                           status_mask);
580        else
581                enable_mask = status_mask << 16;
582        __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
583}
584
585/**
586 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
587 * @dev: drm device
588 */
589static void i915_enable_asle_pipestat(struct drm_device *dev)
590{
591        struct drm_i915_private *dev_priv = dev->dev_private;
592
593        if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
594                return;
595
596        spin_lock_irq(&dev_priv->irq_lock);
597
598        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
599        if (INTEL_INFO(dev)->gen >= 4)
600                i915_enable_pipestat(dev_priv, PIPE_A,
601                                     PIPE_LEGACY_BLC_EVENT_STATUS);
602
603        spin_unlock_irq(&dev_priv->irq_lock);
604}
605
606/*
607 * This timing diagram depicts the video signal in and
608 * around the vertical blanking period.
609 *
610 * Assumptions about the fictitious mode used in this example:
611 *  vblank_start >= 3
612 *  vsync_start = vblank_start + 1
613 *  vsync_end = vblank_start + 2
614 *  vtotal = vblank_start + 3
615 *
616 *           start of vblank:
617 *           latch double buffered registers
618 *           increment frame counter (ctg+)
619 *           generate start of vblank interrupt (gen4+)
620 *           |
621 *           |          frame start:
622 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
623 *           |          may be shifted forward 1-3 extra lines via PIPECONF
624 *           |          |
625 *           |          |  start of vsync:
626 *           |          |  generate vsync interrupt
627 *           |          |  |
628 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
629 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
630 * ----va---> <-----------------vb--------------------> <--------va-------------
631 *       |          |       <----vs----->                     |
632 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
633 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
634 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
635 *       |          |                                         |
636 *       last visible pixel                                   first visible pixel
637 *                  |                                         increment frame counter (gen3/4)
638 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
639 *
640 * x  = horizontal active
641 * _  = horizontal blanking
642 * hs = horizontal sync
643 * va = vertical active
644 * vb = vertical blanking
645 * vs = vertical sync
646 * vbs = vblank_start (number)
647 *
648 * Summary:
649 * - most events happen at the start of horizontal sync
650 * - frame start happens at the start of horizontal blank, 1-4 lines
651 *   (depending on PIPECONF settings) after the start of vblank
652 * - gen3/4 pixel and frame counter are synchronized with the start
653 *   of horizontal active on the first line of vertical active
654 */
655
656static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
657{
658        /* Gen2 doesn't have a hardware frame counter */
659        return 0;
660}
661
662/* Called from drm generic code, passed a 'crtc', which
663 * we use as a pipe index
664 */
665static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
666{
667        struct drm_i915_private *dev_priv = dev->dev_private;
668        unsigned long high_frame;
669        unsigned long low_frame;
670        u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
671        struct intel_crtc *intel_crtc =
672                to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
673        const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
674
675        htotal = mode->crtc_htotal;
676        hsync_start = mode->crtc_hsync_start;
677        vbl_start = mode->crtc_vblank_start;
678        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
679                vbl_start = DIV_ROUND_UP(vbl_start, 2);
680
681        /* Convert to pixel count */
682        vbl_start *= htotal;
683
684        /* Start of vblank event occurs at start of hsync */
685        vbl_start -= htotal - hsync_start;
686
687        high_frame = PIPEFRAME(pipe);
688        low_frame = PIPEFRAMEPIXEL(pipe);
689
690        /*
691         * High & low register fields aren't synchronized, so make sure
692         * we get a low value that's stable across two reads of the high
693         * register.
694         */
695        do {
696                high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
697                low   = I915_READ(low_frame);
698                high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
699        } while (high1 != high2);
700
701        high1 >>= PIPE_FRAME_HIGH_SHIFT;
702        pixel = low & PIPE_PIXEL_MASK;
703        low >>= PIPE_FRAME_LOW_SHIFT;
704
705        /*
706         * The frame counter increments at beginning of active.
707         * Cook up a vblank counter by also checking the pixel
708         * counter against vblank start.
709         */
710        return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
711}
712
713static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
714{
715        struct drm_i915_private *dev_priv = dev->dev_private;
716
717        return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
718}
719
720/* raw reads, only for fast reads of display block, no need for forcewake etc. */
721#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
722
723static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
724{
725        struct drm_device *dev = crtc->base.dev;
726        struct drm_i915_private *dev_priv = dev->dev_private;
727        const struct drm_display_mode *mode = &crtc->base.hwmode;
728        enum pipe pipe = crtc->pipe;
729        int position, vtotal;
730
731        vtotal = mode->crtc_vtotal;
732        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
733                vtotal /= 2;
734
735        if (IS_GEN2(dev))
736                position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
737        else
738                position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
739
740        /*
741         * On HSW, the DSL reg (0x70000) appears to return 0 if we
742         * read it just before the start of vblank.  So try it again
743         * so we don't accidentally end up spanning a vblank frame
744         * increment, causing the pipe_update_end() code to squak at us.
745         *
746         * The nature of this problem means we can't simply check the ISR
747         * bit and return the vblank start value; nor can we use the scanline
748         * debug register in the transcoder as it appears to have the same
749         * problem.  We may need to extend this to include other platforms,
750         * but so far testing only shows the problem on HSW.
751         */
752        if (HAS_DDI(dev) && !position) {
753                int i, temp;
754
755                for (i = 0; i < 100; i++) {
756                        udelay(1);
757                        temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
758                                DSL_LINEMASK_GEN3;
759                        if (temp != position) {
760                                position = temp;
761                                break;
762                        }
763                }
764        }
765
766        /*
767         * See update_scanline_offset() for the details on the
768         * scanline_offset adjustment.
769         */
770        return (position + crtc->scanline_offset) % vtotal;
771}
772
773static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
774                                    unsigned int flags, int *vpos, int *hpos,
775                                    ktime_t *stime, ktime_t *etime,
776                                    const struct drm_display_mode *mode)
777{
778        struct drm_i915_private *dev_priv = dev->dev_private;
779        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
780        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
781        int position;
782        int vbl_start, vbl_end, hsync_start, htotal, vtotal;
783        bool in_vbl = true;
784        int ret = 0;
785        unsigned long irqflags;
786
787        if (WARN_ON(!mode->crtc_clock)) {
788                DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
789                                 "pipe %c\n", pipe_name(pipe));
790                return 0;
791        }
792
793        htotal = mode->crtc_htotal;
794        hsync_start = mode->crtc_hsync_start;
795        vtotal = mode->crtc_vtotal;
796        vbl_start = mode->crtc_vblank_start;
797        vbl_end = mode->crtc_vblank_end;
798
799        if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
800                vbl_start = DIV_ROUND_UP(vbl_start, 2);
801                vbl_end /= 2;
802                vtotal /= 2;
803        }
804
805        ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
806
807        /*
808         * Lock uncore.lock, as we will do multiple timing critical raw
809         * register reads, potentially with preemption disabled, so the
810         * following code must not block on uncore.lock.
811         */
812        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
813
814        /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
815
816        /* Get optional system timestamp before query. */
817        if (stime)
818                *stime = ktime_get();
819
820        if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
821                /* No obvious pixelcount register. Only query vertical
822                 * scanout position from Display scan line register.
823                 */
824                position = __intel_get_crtc_scanline(intel_crtc);
825        } else {
826                /* Have access to pixelcount since start of frame.
827                 * We can split this into vertical and horizontal
828                 * scanout position.
829                 */
830                position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
831
832                /* convert to pixel counts */
833                vbl_start *= htotal;
834                vbl_end *= htotal;
835                vtotal *= htotal;
836
837                /*
838                 * In interlaced modes, the pixel counter counts all pixels,
839                 * so one field will have htotal more pixels. In order to avoid
840                 * the reported position from jumping backwards when the pixel
841                 * counter is beyond the length of the shorter field, just
842                 * clamp the position the length of the shorter field. This
843                 * matches how the scanline counter based position works since
844                 * the scanline counter doesn't count the two half lines.
845                 */
846                if (position >= vtotal)
847                        position = vtotal - 1;
848
849                /*
850                 * Start of vblank interrupt is triggered at start of hsync,
851                 * just prior to the first active line of vblank. However we
852                 * consider lines to start at the leading edge of horizontal
853                 * active. So, should we get here before we've crossed into
854                 * the horizontal active of the first line in vblank, we would
855                 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
856                 * always add htotal-hsync_start to the current pixel position.
857                 */
858                position = (position + htotal - hsync_start) % vtotal;
859        }
860
861        /* Get optional system timestamp after query. */
862        if (etime)
863                *etime = ktime_get();
864
865        /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
866
867        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
868
869        in_vbl = position >= vbl_start && position < vbl_end;
870
871        /*
872         * While in vblank, position will be negative
873         * counting up towards 0 at vbl_end. And outside
874         * vblank, position will be positive counting
875         * up since vbl_end.
876         */
877        if (position >= vbl_start)
878                position -= vbl_end;
879        else
880                position += vtotal - vbl_end;
881
882        if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
883                *vpos = position;
884                *hpos = 0;
885        } else {
886                *vpos = position / htotal;
887                *hpos = position - (*vpos * htotal);
888        }
889
890        /* In vblank? */
891        if (in_vbl)
892                ret |= DRM_SCANOUTPOS_IN_VBLANK;
893
894        return ret;
895}
896
897int intel_get_crtc_scanline(struct intel_crtc *crtc)
898{
899        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
900        unsigned long irqflags;
901        int position;
902
903        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
904        position = __intel_get_crtc_scanline(crtc);
905        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
906
907        return position;
908}
909
910static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
911                              int *max_error,
912                              struct timeval *vblank_time,
913                              unsigned flags)
914{
915        struct drm_crtc *crtc;
916
917        if (pipe >= INTEL_INFO(dev)->num_pipes) {
918                DRM_ERROR("Invalid crtc %u\n", pipe);
919                return -EINVAL;
920        }
921
922        /* Get drm_crtc to timestamp: */
923        crtc = intel_get_crtc_for_pipe(dev, pipe);
924        if (crtc == NULL) {
925                DRM_ERROR("Invalid crtc %u\n", pipe);
926                return -EINVAL;
927        }
928
929        if (!crtc->hwmode.crtc_clock) {
930                DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
931                return -EBUSY;
932        }
933
934        /* Helper routine in DRM core does all the work: */
935        return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
936                                                     vblank_time, flags,
937                                                     &crtc->hwmode);
938}
939
940static void ironlake_rps_change_irq_handler(struct drm_device *dev)
941{
942        struct drm_i915_private *dev_priv = dev->dev_private;
943        u32 busy_up, busy_down, max_avg, min_avg;
944        u8 new_delay;
945
946        spin_lock(&mchdev_lock);
947
948        I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
949
950        new_delay = dev_priv->ips.cur_delay;
951
952        I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
953        busy_up = I915_READ(RCPREVBSYTUPAVG);
954        busy_down = I915_READ(RCPREVBSYTDNAVG);
955        max_avg = I915_READ(RCBMAXAVG);
956        min_avg = I915_READ(RCBMINAVG);
957
958        /* Handle RCS change request from hw */
959        if (busy_up > max_avg) {
960                if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
961                        new_delay = dev_priv->ips.cur_delay - 1;
962                if (new_delay < dev_priv->ips.max_delay)
963                        new_delay = dev_priv->ips.max_delay;
964        } else if (busy_down < min_avg) {
965                if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
966                        new_delay = dev_priv->ips.cur_delay + 1;
967                if (new_delay > dev_priv->ips.min_delay)
968                        new_delay = dev_priv->ips.min_delay;
969        }
970
971        if (ironlake_set_drps(dev, new_delay))
972                dev_priv->ips.cur_delay = new_delay;
973
974        spin_unlock(&mchdev_lock);
975
976        return;
977}
978
979static void notify_ring(struct intel_engine_cs *ring)
980{
981        if (!intel_ring_initialized(ring))
982                return;
983
984        trace_i915_gem_request_notify(ring);
985
986        wake_up_all(&ring->irq_queue);
987}
988
989static void vlv_c0_read(struct drm_i915_private *dev_priv,
990                        struct intel_rps_ei *ei)
991{
992        ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
993        ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
994        ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
995}
996
997static bool vlv_c0_above(struct drm_i915_private *dev_priv,
998                         const struct intel_rps_ei *old,
999                         const struct intel_rps_ei *now,
1000                         int threshold)
1001{
1002        u64 time, c0;
1003        unsigned int mul = 100;
1004
1005        if (old->cz_clock == 0)
1006                return false;
1007
1008        if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
1009                mul <<= 8;
1010
1011        time = now->cz_clock - old->cz_clock;
1012        time *= threshold * dev_priv->czclk_freq;
1013
1014        /* Workload can be split between render + media, e.g. SwapBuffers
1015         * being blitted in X after being rendered in mesa. To account for
1016         * this we need to combine both engines into our activity counter.
1017         */
1018        c0 = now->render_c0 - old->render_c0;
1019        c0 += now->media_c0 - old->media_c0;
1020        c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1021
1022        return c0 >= time;
1023}
1024
1025void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1026{
1027        vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
1028        dev_priv->rps.up_ei = dev_priv->rps.down_ei;
1029}
1030
1031static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
1032{
1033        struct intel_rps_ei now;
1034        u32 events = 0;
1035
1036        if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1037                return 0;
1038
1039        vlv_c0_read(dev_priv, &now);
1040        if (now.cz_clock == 0)
1041                return 0;
1042
1043        if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
1044                if (!vlv_c0_above(dev_priv,
1045                                  &dev_priv->rps.down_ei, &now,
1046                                  dev_priv->rps.down_threshold))
1047                        events |= GEN6_PM_RP_DOWN_THRESHOLD;
1048                dev_priv->rps.down_ei = now;
1049        }
1050
1051        if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
1052                if (vlv_c0_above(dev_priv,
1053                                 &dev_priv->rps.up_ei, &now,
1054                                 dev_priv->rps.up_threshold))
1055                        events |= GEN6_PM_RP_UP_THRESHOLD;
1056                dev_priv->rps.up_ei = now;
1057        }
1058
1059        return events;
1060}
1061
1062static bool any_waiters(struct drm_i915_private *dev_priv)
1063{
1064        struct intel_engine_cs *ring;
1065        int i;
1066
1067        for_each_ring(ring, dev_priv, i)
1068                if (ring->irq_refcount)
1069                        return true;
1070
1071        return false;
1072}
1073
1074static void gen6_pm_rps_work(struct work_struct *work)
1075{
1076        struct drm_i915_private *dev_priv =
1077                container_of(work, struct drm_i915_private, rps.work);
1078        bool client_boost;
1079        int new_delay, adj, min, max;
1080        u32 pm_iir;
1081
1082        spin_lock_irq(&dev_priv->irq_lock);
1083        /* Speed up work cancelation during disabling rps interrupts. */
1084        if (!dev_priv->rps.interrupts_enabled) {
1085                spin_unlock_irq(&dev_priv->irq_lock);
1086                return;
1087        }
1088        pm_iir = dev_priv->rps.pm_iir;
1089        dev_priv->rps.pm_iir = 0;
1090        /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1091        gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1092        client_boost = dev_priv->rps.client_boost;
1093        dev_priv->rps.client_boost = false;
1094        spin_unlock_irq(&dev_priv->irq_lock);
1095
1096        /* Make sure we didn't queue anything we're not going to process. */
1097        WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1098
1099        if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1100                return;
1101
1102        mutex_lock(&dev_priv->rps.hw_lock);
1103
1104        pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);
1105
1106        adj = dev_priv->rps.last_adj;
1107        new_delay = dev_priv->rps.cur_freq;
1108        min = dev_priv->rps.min_freq_softlimit;
1109        max = dev_priv->rps.max_freq_softlimit;
1110
1111        if (client_boost) {
1112                new_delay = dev_priv->rps.max_freq_softlimit;
1113                adj = 0;
1114        } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1115                if (adj > 0)
1116                        adj *= 2;
1117                else /* CHV needs even encode values */
1118                        adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1119                /*
1120                 * For better performance, jump directly
1121                 * to RPe if we're below it.
1122                 */
1123                if (new_delay < dev_priv->rps.efficient_freq - adj) {
1124                        new_delay = dev_priv->rps.efficient_freq;
1125                        adj = 0;
1126                }
1127        } else if (any_waiters(dev_priv)) {
1128                adj = 0;
1129        } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1130                if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1131                        new_delay = dev_priv->rps.efficient_freq;
1132                else
1133                        new_delay = dev_priv->rps.min_freq_softlimit;
1134                adj = 0;
1135        } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1136                if (adj < 0)
1137                        adj *= 2;
1138                else /* CHV needs even encode values */
1139                        adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1140        } else { /* unknown event */
1141                adj = 0;
1142        }
1143
1144        dev_priv->rps.last_adj = adj;
1145
1146        /* sysfs frequency interfaces may have snuck in while servicing the
1147         * interrupt
1148         */
1149        new_delay += adj;
1150        new_delay = clamp_t(int, new_delay, min, max);
1151
1152        intel_set_rps(dev_priv->dev, new_delay);
1153
1154        mutex_unlock(&dev_priv->rps.hw_lock);
1155}
1156
1157
1158/**
1159 * ivybridge_parity_work - Workqueue called when a parity error interrupt
1160 * occurred.
1161 * @work: workqueue struct
1162 *
1163 * Doesn't actually do anything except notify userspace. As a consequence of
1164 * this event, userspace should try to remap the bad rows since statistically
1165 * it is likely the same row is more likely to go bad again.
1166 */
1167static void ivybridge_parity_work(struct work_struct *work)
1168{
1169        struct drm_i915_private *dev_priv =
1170                container_of(work, struct drm_i915_private, l3_parity.error_work);
1171        u32 error_status, row, bank, subbank;
1172        char *parity_event[6];
1173        uint32_t misccpctl;
1174        uint8_t slice = 0;
1175
1176        /* We must turn off DOP level clock gating to access the L3 registers.
1177         * In order to prevent a get/put style interface, acquire struct mutex
1178         * any time we access those registers.
1179         */
1180        mutex_lock(&dev_priv->dev->struct_mutex);
1181
1182        /* If we've screwed up tracking, just let the interrupt fire again */
1183        if (WARN_ON(!dev_priv->l3_parity.which_slice))
1184                goto out;
1185
1186        misccpctl = I915_READ(GEN7_MISCCPCTL);
1187        I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1188        POSTING_READ(GEN7_MISCCPCTL);
1189
1190        while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1191                u32 reg;
1192
1193                slice--;
1194                if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1195                        break;
1196
1197                dev_priv->l3_parity.which_slice &= ~(1<<slice);
1198
1199                reg = GEN7_L3CDERRST1 + (slice * 0x200);
1200
1201                error_status = I915_READ(reg);
1202                row = GEN7_PARITY_ERROR_ROW(error_status);
1203                bank = GEN7_PARITY_ERROR_BANK(error_status);
1204                subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1205
1206                I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1207                POSTING_READ(reg);
1208
1209                parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1210                parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1211                parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1212                parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1213                parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1214                parity_event[5] = NULL;
1215
1216                kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1217                                   KOBJ_CHANGE, parity_event);
1218
1219                DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1220                          slice, row, bank, subbank);
1221
1222                kfree(parity_event[4]);
1223                kfree(parity_event[3]);
1224                kfree(parity_event[2]);
1225                kfree(parity_event[1]);
1226        }
1227
1228        I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1229
1230out:
1231        WARN_ON(dev_priv->l3_parity.which_slice);
1232        spin_lock_irq(&dev_priv->irq_lock);
1233        gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
1234        spin_unlock_irq(&dev_priv->irq_lock);
1235
1236        mutex_unlock(&dev_priv->dev->struct_mutex);
1237}
1238
1239static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1240{
1241        struct drm_i915_private *dev_priv = dev->dev_private;
1242
1243        if (!HAS_L3_DPF(dev))
1244                return;
1245
1246        spin_lock(&dev_priv->irq_lock);
1247        gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1248        spin_unlock(&dev_priv->irq_lock);
1249
1250        iir &= GT_PARITY_ERROR(dev);
1251        if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1252                dev_priv->l3_parity.which_slice |= 1 << 1;
1253
1254        if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1255                dev_priv->l3_parity.which_slice |= 1 << 0;
1256
1257        queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1258}
1259
1260static void ilk_gt_irq_handler(struct drm_device *dev,
1261                               struct drm_i915_private *dev_priv,
1262                               u32 gt_iir)
1263{
1264        if (gt_iir &
1265            (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1266                notify_ring(&dev_priv->ring[RCS]);
1267        if (gt_iir & ILK_BSD_USER_INTERRUPT)
1268                notify_ring(&dev_priv->ring[VCS]);
1269}
1270
1271static void snb_gt_irq_handler(struct drm_device *dev,
1272                               struct drm_i915_private *dev_priv,
1273                               u32 gt_iir)
1274{
1275
1276        if (gt_iir &
1277            (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1278                notify_ring(&dev_priv->ring[RCS]);
1279        if (gt_iir & GT_BSD_USER_INTERRUPT)
1280                notify_ring(&dev_priv->ring[VCS]);
1281        if (gt_iir & GT_BLT_USER_INTERRUPT)
1282                notify_ring(&dev_priv->ring[BCS]);
1283
1284        if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1285                      GT_BSD_CS_ERROR_INTERRUPT |
1286                      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
1287                DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1288
1289        if (gt_iir & GT_PARITY_ERROR(dev))
1290                ivybridge_parity_error_irq_handler(dev, gt_iir);
1291}
1292
1293static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1294                                       u32 master_ctl)
1295{
1296        irqreturn_t ret = IRQ_NONE;
1297
1298        if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1299                u32 tmp = I915_READ_FW(GEN8_GT_IIR(0));
1300                if (tmp) {
1301                        I915_WRITE_FW(GEN8_GT_IIR(0), tmp);
1302                        ret = IRQ_HANDLED;
1303
1304                        if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1305                                intel_lrc_irq_handler(&dev_priv->ring[RCS]);
1306                        if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT))
1307                                notify_ring(&dev_priv->ring[RCS]);
1308
1309                        if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1310                                intel_lrc_irq_handler(&dev_priv->ring[BCS]);
1311                        if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT))
1312                                notify_ring(&dev_priv->ring[BCS]);
1313                } else
1314                        DRM_ERROR("The master control interrupt lied (GT0)!\n");
1315        }
1316
1317        if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1318                u32 tmp = I915_READ_FW(GEN8_GT_IIR(1));
1319                if (tmp) {
1320                        I915_WRITE_FW(GEN8_GT_IIR(1), tmp);
1321                        ret = IRQ_HANDLED;
1322
1323                        if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1324                                intel_lrc_irq_handler(&dev_priv->ring[VCS]);
1325                        if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT))
1326                                notify_ring(&dev_priv->ring[VCS]);
1327
1328                        if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1329                                intel_lrc_irq_handler(&dev_priv->ring[VCS2]);
1330                        if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT))
1331                                notify_ring(&dev_priv->ring[VCS2]);
1332                } else
1333                        DRM_ERROR("The master control interrupt lied (GT1)!\n");
1334        }
1335
1336        if (master_ctl & GEN8_GT_VECS_IRQ) {
1337                u32 tmp = I915_READ_FW(GEN8_GT_IIR(3));
1338                if (tmp) {
1339                        I915_WRITE_FW(GEN8_GT_IIR(3), tmp);
1340                        ret = IRQ_HANDLED;
1341
1342                        if (tmp & (GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1343                                intel_lrc_irq_handler(&dev_priv->ring[VECS]);
1344                        if (tmp & (GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT))
1345                                notify_ring(&dev_priv->ring[VECS]);
1346                } else
1347                        DRM_ERROR("The master control interrupt lied (GT3)!\n");
1348        }
1349
1350        if (master_ctl & GEN8_GT_PM_IRQ) {
1351                u32 tmp = I915_READ_FW(GEN8_GT_IIR(2));
1352                if (tmp & dev_priv->pm_rps_events) {
1353                        I915_WRITE_FW(GEN8_GT_IIR(2),
1354                                      tmp & dev_priv->pm_rps_events);
1355                        ret = IRQ_HANDLED;
1356                        gen6_rps_irq_handler(dev_priv, tmp);
1357                } else
1358                        DRM_ERROR("The master control interrupt lied (PM)!\n");
1359        }
1360
1361        return ret;
1362}
1363
1364static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
1365{
1366        switch (port) {
1367        case PORT_A:
1368                return val & PORTA_HOTPLUG_LONG_DETECT;
1369        case PORT_B:
1370                return val & PORTB_HOTPLUG_LONG_DETECT;
1371        case PORT_C:
1372                return val & PORTC_HOTPLUG_LONG_DETECT;
1373        default:
1374                return false;
1375        }
1376}
1377
1378static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
1379{
1380        switch (port) {
1381        case PORT_E:
1382                return val & PORTE_HOTPLUG_LONG_DETECT;
1383        default:
1384                return false;
1385        }
1386}
1387
1388static bool spt_port_hotplug_long_detect(enum port port, u32 val)
1389{
1390        switch (port) {
1391        case PORT_A:
1392                return val & PORTA_HOTPLUG_LONG_DETECT;
1393        case PORT_B:
1394                return val & PORTB_HOTPLUG_LONG_DETECT;
1395        case PORT_C:
1396                return val & PORTC_HOTPLUG_LONG_DETECT;
1397        case PORT_D:
1398                return val & PORTD_HOTPLUG_LONG_DETECT;
1399        default:
1400                return false;
1401        }
1402}
1403
1404static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
1405{
1406        switch (port) {
1407        case PORT_A:
1408                return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
1409        default:
1410                return false;
1411        }
1412}
1413
1414static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1415{
1416        switch (port) {
1417        case PORT_B:
1418                return val & PORTB_HOTPLUG_LONG_DETECT;
1419        case PORT_C:
1420                return val & PORTC_HOTPLUG_LONG_DETECT;
1421        case PORT_D:
1422                return val & PORTD_HOTPLUG_LONG_DETECT;
1423        default:
1424                return false;
1425        }
1426}
1427
1428static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1429{
1430        switch (port) {
1431        case PORT_B:
1432                return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1433        case PORT_C:
1434                return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1435        case PORT_D:
1436                return val & PORTD_HOTPLUG_INT_LONG_PULSE;
1437        default:
1438                return false;
1439        }
1440}
1441
1442/*
1443 * Get a bit mask of pins that have triggered, and which ones may be long.
1444 * This can be called multiple times with the same masks to accumulate
1445 * hotplug detection results from several registers.
1446 *
1447 * Note that the caller is expected to zero out the masks initially.
1448 */
1449static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1450                             u32 hotplug_trigger, u32 dig_hotplug_reg,
1451                             const u32 hpd[HPD_NUM_PINS],
1452                             bool long_pulse_detect(enum port port, u32 val))
1453{
1454        enum port port;
1455        int i;
1456
1457        for_each_hpd_pin(i) {
1458                if ((hpd[i] & hotplug_trigger) == 0)
1459                        continue;
1460
1461                *pin_mask |= BIT(i);
1462
1463                if (!intel_hpd_pin_to_port(i, &port))
1464                        continue;
1465
1466                if (long_pulse_detect(port, dig_hotplug_reg))
1467                        *long_mask |= BIT(i);
1468        }
1469
1470        DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
1471                         hotplug_trigger, dig_hotplug_reg, *pin_mask);
1472
1473}
1474
1475static void gmbus_irq_handler(struct drm_device *dev)
1476{
1477        struct drm_i915_private *dev_priv = dev->dev_private;
1478
1479        wake_up_all(&dev_priv->gmbus_wait_queue);
1480}
1481
1482static void dp_aux_irq_handler(struct drm_device *dev)
1483{
1484        struct drm_i915_private *dev_priv = dev->dev_private;
1485
1486        wake_up_all(&dev_priv->gmbus_wait_queue);
1487}
1488
1489#if defined(CONFIG_DEBUG_FS)
1490static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1491                                         uint32_t crc0, uint32_t crc1,
1492                                         uint32_t crc2, uint32_t crc3,
1493                                         uint32_t crc4)
1494{
1495        struct drm_i915_private *dev_priv = dev->dev_private;
1496        struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1497        struct intel_pipe_crc_entry *entry;
1498        int head, tail;
1499
1500        spin_lock(&pipe_crc->lock);
1501
1502        if (!pipe_crc->entries) {
1503                spin_unlock(&pipe_crc->lock);
1504                DRM_DEBUG_KMS("spurious interrupt\n");
1505                return;
1506        }
1507
1508        head = pipe_crc->head;
1509        tail = pipe_crc->tail;
1510
1511        if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1512                spin_unlock(&pipe_crc->lock);
1513                DRM_ERROR("CRC buffer overflowing\n");
1514                return;
1515        }
1516
1517        entry = &pipe_crc->entries[head];
1518
1519        entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1520        entry->crc[0] = crc0;
1521        entry->crc[1] = crc1;
1522        entry->crc[2] = crc2;
1523        entry->crc[3] = crc3;
1524        entry->crc[4] = crc4;
1525
1526        head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1527        pipe_crc->head = head;
1528
1529        spin_unlock(&pipe_crc->lock);
1530
1531        wake_up_interruptible(&pipe_crc->wq);
1532}
1533#else
1534static inline void
1535display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1536                             uint32_t crc0, uint32_t crc1,
1537                             uint32_t crc2, uint32_t crc3,
1538                             uint32_t crc4) {}
1539#endif
1540
1541
1542static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1543{
1544        struct drm_i915_private *dev_priv = dev->dev_private;
1545
1546        display_pipe_crc_irq_handler(dev, pipe,
1547                                     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1548                                     0, 0, 0, 0);
1549}
1550
1551static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1552{
1553        struct drm_i915_private *dev_priv = dev->dev_private;
1554
1555        display_pipe_crc_irq_handler(dev, pipe,
1556                                     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1557                                     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1558                                     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1559                                     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1560                                     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1561}
1562
1563static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1564{
1565        struct drm_i915_private *dev_priv = dev->dev_private;
1566        uint32_t res1, res2;
1567
1568        if (INTEL_INFO(dev)->gen >= 3)
1569                res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1570        else
1571                res1 = 0;
1572
1573        if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1574                res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1575        else
1576                res2 = 0;
1577
1578        display_pipe_crc_irq_handler(dev, pipe,
1579                                     I915_READ(PIPE_CRC_RES_RED(pipe)),
1580                                     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1581                                     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1582                                     res1, res2);
1583}
1584
1585/* The RPS events need forcewake, so we add them to a work queue and mask their
1586 * IMR bits until the work is done. Other interrupts can be processed without
1587 * the work queue. */
1588static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1589{
1590        if (pm_iir & dev_priv->pm_rps_events) {
1591                spin_lock(&dev_priv->irq_lock);
1592                gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1593                if (dev_priv->rps.interrupts_enabled) {
1594                        dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1595                        queue_work(dev_priv->wq, &dev_priv->rps.work);
1596                }
1597                spin_unlock(&dev_priv->irq_lock);
1598        }
1599
1600        if (INTEL_INFO(dev_priv)->gen >= 8)
1601                return;
1602
1603        if (HAS_VEBOX(dev_priv->dev)) {
1604                if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1605                        notify_ring(&dev_priv->ring[VECS]);
1606
1607                if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1608                        DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1609        }
1610}
1611
1612static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1613{
1614        if (!drm_handle_vblank(dev, pipe))
1615                return false;
1616
1617        return true;
1618}
1619
1620static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1621{
1622        struct drm_i915_private *dev_priv = dev->dev_private;
1623        u32 pipe_stats[I915_MAX_PIPES] = { };
1624        int pipe;
1625
1626        spin_lock(&dev_priv->irq_lock);
1627        for_each_pipe(dev_priv, pipe) {
1628                int reg;
1629                u32 mask, iir_bit = 0;
1630
1631                /*
1632                 * PIPESTAT bits get signalled even when the interrupt is
1633                 * disabled with the mask bits, and some of the status bits do
1634                 * not generate interrupts at all (like the underrun bit). Hence
1635                 * we need to be careful that we only handle what we want to
1636                 * handle.
1637                 */
1638
1639                /* fifo underruns are filterered in the underrun handler. */
1640                mask = PIPE_FIFO_UNDERRUN_STATUS;
1641
1642                switch (pipe) {
1643                case PIPE_A:
1644                        iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1645                        break;
1646                case PIPE_B:
1647                        iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1648                        break;
1649                case PIPE_C:
1650                        iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1651                        break;
1652                }
1653                if (iir & iir_bit)
1654                        mask |= dev_priv->pipestat_irq_mask[pipe];
1655
1656                if (!mask)
1657                        continue;
1658
1659                reg = PIPESTAT(pipe);
1660                mask |= PIPESTAT_INT_ENABLE_MASK;
1661                pipe_stats[pipe] = I915_READ(reg) & mask;
1662
1663                /*
1664                 * Clear the PIPE*STAT regs before the IIR
1665                 */
1666                if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1667                                        PIPESTAT_INT_STATUS_MASK))
1668                        I915_WRITE(reg, pipe_stats[pipe]);
1669        }
1670        spin_unlock(&dev_priv->irq_lock);
1671
1672        for_each_pipe(dev_priv, pipe) {
1673                if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1674                    intel_pipe_handle_vblank(dev, pipe))
1675                        intel_check_page_flip(dev, pipe);
1676
1677                if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1678                        intel_prepare_page_flip(dev, pipe);
1679                        intel_finish_page_flip(dev, pipe);
1680                }
1681
1682                if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1683                        i9xx_pipe_crc_irq_handler(dev, pipe);
1684
1685                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1686                        intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1687        }
1688
1689        if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1690                gmbus_irq_handler(dev);
1691}
1692
1693static void i9xx_hpd_irq_handler(struct drm_device *dev)
1694{
1695        struct drm_i915_private *dev_priv = dev->dev_private;
1696        u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1697        u32 pin_mask = 0, long_mask = 0;
1698
1699        if (!hotplug_status)
1700                return;
1701
1702        I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1703        /*
1704         * Make sure hotplug status is cleared before we clear IIR, or else we
1705         * may miss hotplug events.
1706         */
1707        POSTING_READ(PORT_HOTPLUG_STAT);
1708
1709        if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
1710                u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1711
1712                if (hotplug_trigger) {
1713                        intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1714                                           hotplug_trigger, hpd_status_g4x,
1715                                           i9xx_port_hotplug_long_detect);
1716
1717                        intel_hpd_irq_handler(dev, pin_mask, long_mask);
1718                }
1719
1720                if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1721                        dp_aux_irq_handler(dev);
1722        } else {
1723                u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1724
1725                if (hotplug_trigger) {
1726                        intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1727                                           hotplug_trigger, hpd_status_i915,
1728                                           i9xx_port_hotplug_long_detect);
1729                        intel_hpd_irq_handler(dev, pin_mask, long_mask);
1730                }
1731        }
1732}
1733
1734static irqreturn_t valleyview_irq_handler(int irq, void *arg)
1735{
1736        struct drm_device *dev = arg;
1737        struct drm_i915_private *dev_priv = dev->dev_private;
1738        u32 iir, gt_iir, pm_iir;
1739        irqreturn_t ret = IRQ_NONE;
1740
1741        if (!intel_irqs_enabled(dev_priv))
1742                return IRQ_NONE;
1743
1744        while (true) {
1745                /* Find, clear, then process each source of interrupt */
1746
1747                gt_iir = I915_READ(GTIIR);
1748                if (gt_iir)
1749                        I915_WRITE(GTIIR, gt_iir);
1750
1751                pm_iir = I915_READ(GEN6_PMIIR);
1752                if (pm_iir)
1753                        I915_WRITE(GEN6_PMIIR, pm_iir);
1754
1755                iir = I915_READ(VLV_IIR);
1756                if (iir) {
1757                        /* Consume port before clearing IIR or we'll miss events */
1758                        if (iir & I915_DISPLAY_PORT_INTERRUPT)
1759                                i9xx_hpd_irq_handler(dev);
1760                        I915_WRITE(VLV_IIR, iir);
1761                }
1762
1763                if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1764                        goto out;
1765
1766                ret = IRQ_HANDLED;
1767
1768                if (gt_iir)
1769                        snb_gt_irq_handler(dev, dev_priv, gt_iir);
1770                if (pm_iir)
1771                        gen6_rps_irq_handler(dev_priv, pm_iir);
1772                /* Call regardless, as some status bits might not be
1773                 * signalled in iir */
1774                valleyview_pipestat_irq_handler(dev, iir);
1775        }
1776
1777out:
1778        return ret;
1779}
1780
1781static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1782{
1783        struct drm_device *dev = arg;
1784        struct drm_i915_private *dev_priv = dev->dev_private;
1785        u32 master_ctl, iir;
1786        irqreturn_t ret = IRQ_NONE;
1787
1788        if (!intel_irqs_enabled(dev_priv))
1789                return IRQ_NONE;
1790
1791        for (;;) {
1792                master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1793                iir = I915_READ(VLV_IIR);
1794
1795                if (master_ctl == 0 && iir == 0)
1796                        break;
1797
1798                ret = IRQ_HANDLED;
1799
1800                I915_WRITE(GEN8_MASTER_IRQ, 0);
1801
1802                /* Find, clear, then process each source of interrupt */
1803
1804                if (iir) {
1805                        /* Consume port before clearing IIR or we'll miss events */
1806                        if (iir & I915_DISPLAY_PORT_INTERRUPT)
1807                                i9xx_hpd_irq_handler(dev);
1808                        I915_WRITE(VLV_IIR, iir);
1809                }
1810
1811                gen8_gt_irq_handler(dev_priv, master_ctl);
1812
1813                /* Call regardless, as some status bits might not be
1814                 * signalled in iir */
1815                valleyview_pipestat_irq_handler(dev, iir);
1816
1817                I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1818                POSTING_READ(GEN8_MASTER_IRQ);
1819        }
1820
1821        return ret;
1822}
1823
1824static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
1825                                const u32 hpd[HPD_NUM_PINS])
1826{
1827        struct drm_i915_private *dev_priv = to_i915(dev);
1828        u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
1829
1830        dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1831        I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1832
1833        intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1834                           dig_hotplug_reg, hpd,
1835                           pch_port_hotplug_long_detect);
1836
1837        intel_hpd_irq_handler(dev, pin_mask, long_mask);
1838}
1839
1840static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1841{
1842        struct drm_i915_private *dev_priv = dev->dev_private;
1843        int pipe;
1844        u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1845
1846        if (hotplug_trigger)
1847                ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1848
1849        if (pch_iir & SDE_AUDIO_POWER_MASK) {
1850                int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1851                               SDE_AUDIO_POWER_SHIFT);
1852                DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1853                                 port_name(port));
1854        }
1855
1856        if (pch_iir & SDE_AUX_MASK)
1857                dp_aux_irq_handler(dev);
1858
1859        if (pch_iir & SDE_GMBUS)
1860                gmbus_irq_handler(dev);
1861
1862        if (pch_iir & SDE_AUDIO_HDCP_MASK)
1863                DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1864
1865        if (pch_iir & SDE_AUDIO_TRANS_MASK)
1866                DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1867
1868        if (pch_iir & SDE_POISON)
1869                DRM_ERROR("PCH poison interrupt\n");
1870
1871        if (pch_iir & SDE_FDI_MASK)
1872                for_each_pipe(dev_priv, pipe)
1873                        DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1874                                         pipe_name(pipe),
1875                                         I915_READ(FDI_RX_IIR(pipe)));
1876
1877        if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1878                DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1879
1880        if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1881                DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1882
1883        if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1884                intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1885
1886        if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1887                intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1888}
1889
1890static void ivb_err_int_handler(struct drm_device *dev)
1891{
1892        struct drm_i915_private *dev_priv = dev->dev_private;
1893        u32 err_int = I915_READ(GEN7_ERR_INT);
1894        enum pipe pipe;
1895
1896        if (err_int & ERR_INT_POISON)
1897                DRM_ERROR("Poison interrupt\n");
1898
1899        for_each_pipe(dev_priv, pipe) {
1900                if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1901                        intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1902
1903                if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1904                        if (IS_IVYBRIDGE(dev))
1905                                ivb_pipe_crc_irq_handler(dev, pipe);
1906                        else
1907                                hsw_pipe_crc_irq_handler(dev, pipe);
1908                }
1909        }
1910
1911        I915_WRITE(GEN7_ERR_INT, err_int);
1912}
1913
1914static void cpt_serr_int_handler(struct drm_device *dev)
1915{
1916        struct drm_i915_private *dev_priv = dev->dev_private;
1917        u32 serr_int = I915_READ(SERR_INT);
1918
1919        if (serr_int & SERR_INT_POISON)
1920                DRM_ERROR("PCH poison interrupt\n");
1921
1922        if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1923                intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1924
1925        if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1926                intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1927
1928        if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1929                intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1930
1931        I915_WRITE(SERR_INT, serr_int);
1932}
1933
1934static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
1935{
1936        struct drm_i915_private *dev_priv = dev->dev_private;
1937        int pipe;
1938        u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1939
1940        if (hotplug_trigger)
1941                ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1942
1943        if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
1944                int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
1945                               SDE_AUDIO_POWER_SHIFT_CPT);
1946                DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
1947                                 port_name(port));
1948        }
1949
1950        if (pch_iir & SDE_AUX_MASK_CPT)
1951                dp_aux_irq_handler(dev);
1952
1953        if (pch_iir & SDE_GMBUS_CPT)
1954                gmbus_irq_handler(dev);
1955
1956        if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
1957                DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
1958
1959        if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
1960                DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
1961
1962        if (pch_iir & SDE_FDI_MASK_CPT)
1963                for_each_pipe(dev_priv, pipe)
1964                        DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
1965                                         pipe_name(pipe),
1966                                         I915_READ(FDI_RX_IIR(pipe)));
1967
1968        if (pch_iir & SDE_ERROR_CPT)
1969                cpt_serr_int_handler(dev);
1970}
1971
1972static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
1973{
1974        struct drm_i915_private *dev_priv = dev->dev_private;
1975        u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
1976                ~SDE_PORTE_HOTPLUG_SPT;
1977        u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
1978        u32 pin_mask = 0, long_mask = 0;
1979
1980        if (hotplug_trigger) {
1981                u32 dig_hotplug_reg;
1982
1983                dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1984                I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1985
1986                intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1987                                   dig_hotplug_reg, hpd_spt,
1988                                   spt_port_hotplug_long_detect);
1989        }
1990
1991        if (hotplug2_trigger) {
1992                u32 dig_hotplug_reg;
1993
1994                dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
1995                I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);
1996
1997                intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
1998                                   dig_hotplug_reg, hpd_spt,
1999                                   spt_port_hotplug2_long_detect);
2000        }
2001
2002        if (pin_mask)
2003                intel_hpd_irq_handler(dev, pin_mask, long_mask);
2004
2005        if (pch_iir & SDE_GMBUS_CPT)
2006                gmbus_irq_handler(dev);
2007}
2008
2009static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2010                                const u32 hpd[HPD_NUM_PINS])
2011{
2012        struct drm_i915_private *dev_priv = to_i915(dev);
2013        u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2014
2015        dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
2016        I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);
2017
2018        intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2019                           dig_hotplug_reg, hpd,
2020                           ilk_port_hotplug_long_detect);
2021
2022        intel_hpd_irq_handler(dev, pin_mask, long_mask);
2023}
2024
2025static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
2026{
2027        struct drm_i915_private *dev_priv = dev->dev_private;
2028        enum pipe pipe;
2029        u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
2030
2031        if (hotplug_trigger)
2032                ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2033
2034        if (de_iir & DE_AUX_CHANNEL_A)
2035                dp_aux_irq_handler(dev);
2036
2037        if (de_iir & DE_GSE)
2038                intel_opregion_asle_intr(dev);
2039
2040        if (de_iir & DE_POISON)
2041                DRM_ERROR("Poison interrupt\n");
2042
2043        for_each_pipe(dev_priv, pipe) {
2044                if (de_iir & DE_PIPE_VBLANK(pipe) &&
2045                    intel_pipe_handle_vblank(dev, pipe))
2046                        intel_check_page_flip(dev, pipe);
2047
2048                if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2049                        intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2050
2051                if (de_iir & DE_PIPE_CRC_DONE(pipe))
2052                        i9xx_pipe_crc_irq_handler(dev, pipe);
2053
2054                /* plane/pipes map 1:1 on ilk+ */
2055                if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
2056                        intel_prepare_page_flip(dev, pipe);
2057                        intel_finish_page_flip_plane(dev, pipe);
2058                }
2059        }
2060
2061        /* check event from PCH */
2062        if (de_iir & DE_PCH_EVENT) {
2063                u32 pch_iir = I915_READ(SDEIIR);
2064
2065                if (HAS_PCH_CPT(dev))
2066                        cpt_irq_handler(dev, pch_iir);
2067                else
2068                        ibx_irq_handler(dev, pch_iir);
2069
2070                /* should clear PCH hotplug event before clear CPU irq */
2071                I915_WRITE(SDEIIR, pch_iir);
2072        }
2073
2074        if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
2075                ironlake_rps_change_irq_handler(dev);
2076}
2077
2078static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
2079{
2080        struct drm_i915_private *dev_priv = dev->dev_private;
2081        enum pipe pipe;
2082        u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
2083
2084        if (hotplug_trigger)
2085                ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2086
2087        if (de_iir & DE_ERR_INT_IVB)
2088                ivb_err_int_handler(dev);
2089
2090        if (de_iir & DE_AUX_CHANNEL_A_IVB)
2091                dp_aux_irq_handler(dev);
2092
2093        if (de_iir & DE_GSE_IVB)
2094                intel_opregion_asle_intr(dev);
2095
2096        for_each_pipe(dev_priv, pipe) {
2097                if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
2098                    intel_pipe_handle_vblank(dev, pipe))
2099                        intel_check_page_flip(dev, pipe);
2100
2101                /* plane/pipes map 1:1 on ilk+ */
2102                if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
2103                        intel_prepare_page_flip(dev, pipe);
2104                        intel_finish_page_flip_plane(dev, pipe);
2105                }
2106        }
2107
2108        /* check event from PCH */
2109        if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
2110                u32 pch_iir = I915_READ(SDEIIR);
2111
2112                cpt_irq_handler(dev, pch_iir);
2113
2114                /* clear PCH hotplug event before clear CPU irq */
2115                I915_WRITE(SDEIIR, pch_iir);
2116        }
2117}
2118
2119/*
2120 * To handle irqs with the minimum potential races with fresh interrupts, we:
2121 * 1 - Disable Master Interrupt Control.
2122 * 2 - Find the source(s) of the interrupt.
2123 * 3 - Clear the Interrupt Identity bits (IIR).
2124 * 4 - Process the interrupt(s) that had bits set in the IIRs.
2125 * 5 - Re-enable Master Interrupt Control.
2126 */
2127static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2128{
2129        struct drm_device *dev = arg;
2130        struct drm_i915_private *dev_priv = dev->dev_private;
2131        u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2132        irqreturn_t ret = IRQ_NONE;
2133
2134        if (!intel_irqs_enabled(dev_priv))
2135                return IRQ_NONE;
2136
2137        /* We get interrupts on unclaimed registers, so check for this before we
2138         * do any I915_{READ,WRITE}. */
2139        intel_uncore_check_errors(dev);
2140
2141        /* disable master interrupt before clearing iir  */
2142        de_ier = I915_READ(DEIER);
2143        I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2144        POSTING_READ(DEIER);
2145
2146        /* Disable south interrupts. We'll only write to SDEIIR once, so further
2147         * interrupts will will be stored on its back queue, and then we'll be
2148         * able to process them after we restore SDEIER (as soon as we restore
2149         * it, we'll get an interrupt if SDEIIR still has something to process
2150         * due to its back queue). */
2151        if (!HAS_PCH_NOP(dev)) {
2152                sde_ier = I915_READ(SDEIER);
2153                I915_WRITE(SDEIER, 0);
2154                POSTING_READ(SDEIER);
2155        }
2156
2157        /* Find, clear, then process each source of interrupt */
2158
2159        gt_iir = I915_READ(GTIIR);
2160        if (gt_iir) {
2161                I915_WRITE(GTIIR, gt_iir);
2162                ret = IRQ_HANDLED;
2163                if (INTEL_INFO(dev)->gen >= 6)
2164                        snb_gt_irq_handler(dev, dev_priv, gt_iir);
2165                else
2166                        ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2167        }
2168
2169        de_iir = I915_READ(DEIIR);
2170        if (de_iir) {
2171                I915_WRITE(DEIIR, de_iir);
2172                ret = IRQ_HANDLED;
2173                if (INTEL_INFO(dev)->gen >= 7)
2174                        ivb_display_irq_handler(dev, de_iir);
2175                else
2176                        ilk_display_irq_handler(dev, de_iir);
2177        }
2178
2179        if (INTEL_INFO(dev)->gen >= 6) {
2180                u32 pm_iir = I915_READ(GEN6_PMIIR);
2181                if (pm_iir) {
2182                        I915_WRITE(GEN6_PMIIR, pm_iir);
2183                        ret = IRQ_HANDLED;
2184                        gen6_rps_irq_handler(dev_priv, pm_iir);
2185                }
2186        }
2187
2188        I915_WRITE(DEIER, de_ier);
2189        POSTING_READ(DEIER);
2190        if (!HAS_PCH_NOP(dev)) {
2191                I915_WRITE(SDEIER, sde_ier);
2192                POSTING_READ(SDEIER);
2193        }
2194
2195        return ret;
2196}
2197
2198static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
2199                                const u32 hpd[HPD_NUM_PINS])
2200{
2201        struct drm_i915_private *dev_priv = to_i915(dev);
2202        u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2203
2204        dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2205        I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2206
2207        intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2208                           dig_hotplug_reg, hpd,
2209                           bxt_port_hotplug_long_detect);
2210
2211        intel_hpd_irq_handler(dev, pin_mask, long_mask);
2212}
2213
2214static irqreturn_t gen8_irq_handler(int irq, void *arg)
2215{
2216        struct drm_device *dev = arg;
2217        struct drm_i915_private *dev_priv = dev->dev_private;
2218        u32 master_ctl;
2219        irqreturn_t ret = IRQ_NONE;
2220        uint32_t tmp = 0;
2221        enum pipe pipe;
2222        u32 aux_mask = GEN8_AUX_CHANNEL_A;
2223
2224        if (!intel_irqs_enabled(dev_priv))
2225                return IRQ_NONE;
2226
2227        if (INTEL_INFO(dev_priv)->gen >= 9)
2228                aux_mask |=  GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2229                        GEN9_AUX_CHANNEL_D;
2230
2231        master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
2232        master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2233        if (!master_ctl)
2234                return IRQ_NONE;
2235
2236        I915_WRITE_FW(GEN8_MASTER_IRQ, 0);
2237
2238        /* Find, clear, then process each source of interrupt */
2239
2240        ret = gen8_gt_irq_handler(dev_priv, master_ctl);
2241
2242        if (master_ctl & GEN8_DE_MISC_IRQ) {
2243                tmp = I915_READ(GEN8_DE_MISC_IIR);
2244                if (tmp) {
2245                        I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2246                        ret = IRQ_HANDLED;
2247                        if (tmp & GEN8_DE_MISC_GSE)
2248                                intel_opregion_asle_intr(dev);
2249                        else
2250                                DRM_ERROR("Unexpected DE Misc interrupt\n");
2251                }
2252                else
2253                        DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2254        }
2255
2256        if (master_ctl & GEN8_DE_PORT_IRQ) {
2257                tmp = I915_READ(GEN8_DE_PORT_IIR);
2258                if (tmp) {
2259                        bool found = false;
2260                        u32 hotplug_trigger = 0;
2261
2262                        if (IS_BROXTON(dev_priv))
2263                                hotplug_trigger = tmp & BXT_DE_PORT_HOTPLUG_MASK;
2264                        else if (IS_BROADWELL(dev_priv))
2265                                hotplug_trigger = tmp & GEN8_PORT_DP_A_HOTPLUG;
2266
2267                        I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2268                        ret = IRQ_HANDLED;
2269
2270                        if (tmp & aux_mask) {
2271                                dp_aux_irq_handler(dev);
2272                                found = true;
2273                        }
2274
2275                        if (hotplug_trigger) {
2276                                if (IS_BROXTON(dev))
2277                                        bxt_hpd_irq_handler(dev, hotplug_trigger, hpd_bxt);
2278                                else
2279                                        ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_bdw);
2280                                found = true;
2281                        }
2282
2283                        if (IS_BROXTON(dev) && (tmp & BXT_DE_PORT_GMBUS)) {
2284                                gmbus_irq_handler(dev);
2285                                found = true;
2286                        }
2287
2288                        if (!found)
2289                                DRM_ERROR("Unexpected DE Port interrupt\n");
2290                }
2291                else
2292                        DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2293        }
2294
2295        for_each_pipe(dev_priv, pipe) {
2296                uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
2297
2298                if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2299                        continue;
2300
2301                pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
2302                if (pipe_iir) {
2303                        ret = IRQ_HANDLED;
2304                        I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
2305
2306                        if (pipe_iir & GEN8_PIPE_VBLANK &&
2307                            intel_pipe_handle_vblank(dev, pipe))
2308                                intel_check_page_flip(dev, pipe);
2309
2310                        if (INTEL_INFO(dev_priv)->gen >= 9)
2311                                flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2312                        else
2313                                flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2314
2315                        if (flip_done) {
2316                                intel_prepare_page_flip(dev, pipe);
2317                                intel_finish_page_flip_plane(dev, pipe);
2318                        }
2319
2320                        if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2321                                hsw_pipe_crc_irq_handler(dev, pipe);
2322
2323                        if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2324                                intel_cpu_fifo_underrun_irq_handler(dev_priv,
2325                                                                    pipe);
2326
2327
2328                        if (INTEL_INFO(dev_priv)->gen >= 9)
2329                                fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2330                        else
2331                                fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2332
2333                        if (fault_errors)
2334                                DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2335                                          pipe_name(pipe),
2336                                          pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
2337                } else
2338                        DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2339        }
2340
2341        if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
2342            master_ctl & GEN8_DE_PCH_IRQ) {
2343                /*
2344                 * FIXME(BDW): Assume for now that the new interrupt handling
2345                 * scheme also closed the SDE interrupt handling race we've seen
2346                 * on older pch-split platforms. But this needs testing.
2347                 */
2348                u32 pch_iir = I915_READ(SDEIIR);
2349                if (pch_iir) {
2350                        I915_WRITE(SDEIIR, pch_iir);
2351                        ret = IRQ_HANDLED;
2352
2353                        if (HAS_PCH_SPT(dev_priv))
2354                                spt_irq_handler(dev, pch_iir);
2355                        else
2356                                cpt_irq_handler(dev, pch_iir);
2357                } else {
2358                        /*
2359                         * Like on previous PCH there seems to be something
2360                         * fishy going on with forwarding PCH interrupts.
2361                         */
2362                        DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
2363                }
2364        }
2365
2366        I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2367        POSTING_READ_FW(GEN8_MASTER_IRQ);
2368
2369        return ret;
2370}
2371
2372static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2373                               bool reset_completed)
2374{
2375        struct intel_engine_cs *ring;
2376        int i;
2377
2378        /*
2379         * Notify all waiters for GPU completion events that reset state has
2380         * been changed, and that they need to restart their wait after
2381         * checking for potential errors (and bail out to drop locks if there is
2382         * a gpu reset pending so that i915_error_work_func can acquire them).
2383         */
2384
2385        /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2386        for_each_ring(ring, dev_priv, i)
2387                wake_up_all(&ring->irq_queue);
2388
2389        /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2390        wake_up_all(&dev_priv->pending_flip_queue);
2391
2392        /*
2393         * Signal tasks blocked in i915_gem_wait_for_error that the pending
2394         * reset state is cleared.
2395         */
2396        if (reset_completed)
2397                wake_up_all(&dev_priv->gpu_error.reset_queue);
2398}
2399
2400/**
2401 * i915_reset_and_wakeup - do process context error handling work
2402 * @dev: drm device
2403 *
2404 * Fire an error uevent so userspace can see that a hang or error
2405 * was detected.
2406 */
2407static void i915_reset_and_wakeup(struct drm_device *dev)
2408{
2409        struct drm_i915_private *dev_priv = to_i915(dev);
2410        struct i915_gpu_error *error = &dev_priv->gpu_error;
2411        char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2412        char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2413        char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2414        int ret;
2415
2416        kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2417
2418        /*
2419         * Note that there's only one work item which does gpu resets, so we
2420         * need not worry about concurrent gpu resets potentially incrementing
2421         * error->reset_counter twice. We only need to take care of another
2422         * racing irq/hangcheck declaring the gpu dead for a second time. A
2423         * quick check for that is good enough: schedule_work ensures the
2424         * correct ordering between hang detection and this work item, and since
2425         * the reset in-progress bit is only ever set by code outside of this
2426         * work we don't need to worry about any other races.
2427         */
2428        if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2429                DRM_DEBUG_DRIVER("resetting chip\n");
2430                kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2431                                   reset_event);
2432
2433                /*
2434                 * In most cases it's guaranteed that we get here with an RPM
2435                 * reference held, for example because there is a pending GPU
2436                 * request that won't finish until the reset is done. This
2437                 * isn't the case at least when we get here by doing a
2438                 * simulated reset via debugs, so get an RPM reference.
2439                 */
2440                intel_runtime_pm_get(dev_priv);
2441
2442                intel_prepare_reset(dev);
2443
2444                /*
2445                 * All state reset _must_ be completed before we update the
2446                 * reset counter, for otherwise waiters might miss the reset
2447                 * pending state and not properly drop locks, resulting in
2448                 * deadlocks with the reset work.
2449                 */
2450                ret = i915_reset(dev);
2451
2452                intel_finish_reset(dev);
2453
2454                intel_runtime_pm_put(dev_priv);
2455
2456                if (ret == 0) {
2457                        /*
2458                         * After all the gem state is reset, increment the reset
2459                         * counter and wake up everyone waiting for the reset to
2460                         * complete.
2461                         *
2462                         * Since unlock operations are a one-sided barrier only,
2463                         * we need to insert a barrier here to order any seqno
2464                         * updates before
2465                         * the counter increment.
2466                         */
2467                        smp_mb__before_atomic();
2468                        atomic_inc(&dev_priv->gpu_error.reset_counter);
2469
2470                        kobject_uevent_env(&dev->primary->kdev->kobj,
2471                                           KOBJ_CHANGE, reset_done_event);
2472                } else {
2473                        atomic_or(I915_WEDGED, &error->reset_counter);
2474                }
2475
2476                /*
2477                 * Note: The wake_up also serves as a memory barrier so that
2478                 * waiters see the update value of the reset counter atomic_t.
2479                 */
2480                i915_error_wake_up(dev_priv, true);
2481        }
2482}
2483
2484static void i915_report_and_clear_eir(struct drm_device *dev)
2485{
2486        struct drm_i915_private *dev_priv = dev->dev_private;
2487        uint32_t instdone[I915_NUM_INSTDONE_REG];
2488        u32 eir = I915_READ(EIR);
2489        int pipe, i;
2490
2491        if (!eir)
2492                return;
2493
2494        pr_err("render error detected, EIR: 0x%08x\n", eir);
2495
2496        i915_get_extra_instdone(dev, instdone);
2497
2498        if (IS_G4X(dev)) {
2499                if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2500                        u32 ipeir = I915_READ(IPEIR_I965);
2501
2502                        pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2503                        pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2504                        for (i = 0; i < ARRAY_SIZE(instdone); i++)
2505                                pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2506                        pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2507                        pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2508                        I915_WRITE(IPEIR_I965, ipeir);
2509                        POSTING_READ(IPEIR_I965);
2510                }
2511                if (eir & GM45_ERROR_PAGE_TABLE) {
2512                        u32 pgtbl_err = I915_READ(PGTBL_ER);
2513                        pr_err("page table error\n");
2514                        pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2515                        I915_WRITE(PGTBL_ER, pgtbl_err);
2516                        POSTING_READ(PGTBL_ER);
2517                }
2518        }
2519
2520        if (!IS_GEN2(dev)) {
2521                if (eir & I915_ERROR_PAGE_TABLE) {
2522                        u32 pgtbl_err = I915_READ(PGTBL_ER);
2523                        pr_err("page table error\n");
2524                        pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2525                        I915_WRITE(PGTBL_ER, pgtbl_err);
2526                        POSTING_READ(PGTBL_ER);
2527                }
2528        }
2529
2530        if (eir & I915_ERROR_MEMORY_REFRESH) {
2531                pr_err("memory refresh error:\n");
2532                for_each_pipe(dev_priv, pipe)
2533                        pr_err("pipe %c stat: 0x%08x\n",
2534                               pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2535                /* pipestat has already been acked */
2536        }
2537        if (eir & I915_ERROR_INSTRUCTION) {
2538                pr_err("instruction error\n");
2539                pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2540                for (i = 0; i < ARRAY_SIZE(instdone); i++)
2541                        pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2542                if (INTEL_INFO(dev)->gen < 4) {
2543                        u32 ipeir = I915_READ(IPEIR);
2544
2545                        pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
2546                        pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
2547                        pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2548                        I915_WRITE(IPEIR, ipeir);
2549                        POSTING_READ(IPEIR);
2550                } else {
2551                        u32 ipeir = I915_READ(IPEIR_I965);
2552
2553                        pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2554                        pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2555                        pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
2556                        pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2557                        I915_WRITE(IPEIR_I965, ipeir);
2558                        POSTING_READ(IPEIR_I965);
2559                }
2560        }
2561
2562        I915_WRITE(EIR, eir);
2563        POSTING_READ(EIR);
2564        eir = I915_READ(EIR);
2565        if (eir) {
2566                /*
2567                 * some errors might have become stuck,
2568                 * mask them.
2569                 */
2570                DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2571                I915_WRITE(EMR, I915_READ(EMR) | eir);
2572                I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2573        }
2574}
2575
2576/**
2577 * i915_handle_error - handle a gpu error
2578 * @dev: drm device
2579 *
2580 * Do some basic checking of register state at error time and
2581 * dump it to the syslog.  Also call i915_capture_error_state() to make
2582 * sure we get a record and make it available in debugfs.  Fire a uevent
2583 * so userspace knows something bad happened (should trigger collection
2584 * of a ring dump etc.).
2585 */
2586void i915_handle_error(struct drm_device *dev, bool wedged,
2587                       const char *fmt, ...)
2588{
2589        struct drm_i915_private *dev_priv = dev->dev_private;
2590        va_list args;
2591        char error_msg[80];
2592
2593        va_start(args, fmt);
2594        vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2595        va_end(args);
2596
2597        i915_capture_error_state(dev, wedged, error_msg);
2598        i915_report_and_clear_eir(dev);
2599
2600        if (wedged) {
2601                atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2602                                &dev_priv->gpu_error.reset_counter);
2603
2604                /*
2605                 * Wakeup waiting processes so that the reset function
2606                 * i915_reset_and_wakeup doesn't deadlock trying to grab
2607                 * various locks. By bumping the reset counter first, the woken
2608                 * processes will see a reset in progress and back off,
2609                 * releasing their locks and then wait for the reset completion.
2610                 * We must do this for _all_ gpu waiters that might hold locks
2611                 * that the reset work needs to acquire.
2612                 *
2613                 * Note: The wake_up serves as the required memory barrier to
2614                 * ensure that the waiters see the updated value of the reset
2615                 * counter atomic_t.
2616                 */
2617                i915_error_wake_up(dev_priv, false);
2618        }
2619
2620        i915_reset_and_wakeup(dev);
2621}
2622
2623/* Called from drm generic code, passed 'crtc' which
2624 * we use as a pipe index
2625 */
2626static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
2627{
2628        struct drm_i915_private *dev_priv = dev->dev_private;
2629        unsigned long irqflags;
2630
2631        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2632        if (INTEL_INFO(dev)->gen >= 4)
2633                i915_enable_pipestat(dev_priv, pipe,
2634                                     PIPE_START_VBLANK_INTERRUPT_STATUS);
2635        else
2636                i915_enable_pipestat(dev_priv, pipe,
2637                                     PIPE_VBLANK_INTERRUPT_STATUS);
2638        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2639
2640        return 0;
2641}
2642
2643static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
2644{
2645        struct drm_i915_private *dev_priv = dev->dev_private;
2646        unsigned long irqflags;
2647        uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2648                                                     DE_PIPE_VBLANK(pipe);
2649
2650        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2651        ironlake_enable_display_irq(dev_priv, bit);
2652        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2653
2654        return 0;
2655}
2656
2657static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
2658{
2659        struct drm_i915_private *dev_priv = dev->dev_private;
2660        unsigned long irqflags;
2661
2662        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2663        i915_enable_pipestat(dev_priv, pipe,
2664                             PIPE_START_VBLANK_INTERRUPT_STATUS);
2665        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2666
2667        return 0;
2668}
2669
2670static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2671{
2672        struct drm_i915_private *dev_priv = dev->dev_private;
2673        unsigned long irqflags;
2674
2675        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2676        dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2677        I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2678        POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2679        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2680        return 0;
2681}
2682
2683/* Called from drm generic code, passed 'crtc' which
2684 * we use as a pipe index
2685 */
2686static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
2687{
2688        struct drm_i915_private *dev_priv = dev->dev_private;
2689        unsigned long irqflags;
2690
2691        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2692        i915_disable_pipestat(dev_priv, pipe,
2693                              PIPE_VBLANK_INTERRUPT_STATUS |
2694                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2695        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2696}
2697
2698static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
2699{
2700        struct drm_i915_private *dev_priv = dev->dev_private;
2701        unsigned long irqflags;
2702        uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2703                                                     DE_PIPE_VBLANK(pipe);
2704
2705        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2706        ironlake_disable_display_irq(dev_priv, bit);
2707        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2708}
2709
2710static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
2711{
2712        struct drm_i915_private *dev_priv = dev->dev_private;
2713        unsigned long irqflags;
2714
2715        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2716        i915_disable_pipestat(dev_priv, pipe,
2717                              PIPE_START_VBLANK_INTERRUPT_STATUS);
2718        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2719}
2720
2721static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2722{
2723        struct drm_i915_private *dev_priv = dev->dev_private;
2724        unsigned long irqflags;
2725
2726        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2727        dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2728        I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2729        POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2730        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2731}
2732
2733static bool
2734ring_idle(struct intel_engine_cs *ring, u32 seqno)
2735{
2736        return (list_empty(&ring->request_list) ||
2737                i915_seqno_passed(seqno, ring->last_submitted_seqno));
2738}
2739
2740static bool
2741ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2742{
2743        if (INTEL_INFO(dev)->gen >= 8) {
2744                return (ipehr >> 23) == 0x1c;
2745        } else {
2746                ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2747                return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2748                                 MI_SEMAPHORE_REGISTER);
2749        }
2750}
2751
2752static struct intel_engine_cs *
2753semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
2754{
2755        struct drm_i915_private *dev_priv = ring->dev->dev_private;
2756        struct intel_engine_cs *signaller;
2757        int i;
2758
2759        if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
2760                for_each_ring(signaller, dev_priv, i) {
2761                        if (ring == signaller)
2762                                continue;
2763
2764                        if (offset == signaller->semaphore.signal_ggtt[ring->id])
2765                                return signaller;
2766                }
2767        } else {
2768                u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2769
2770                for_each_ring(signaller, dev_priv, i) {
2771                        if(ring == signaller)
2772                                continue;
2773
2774                        if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2775                                return signaller;
2776                }
2777        }
2778
2779        DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2780                  ring->id, ipehr, offset);
2781
2782        return NULL;
2783}
2784
2785static struct intel_engine_cs *
2786semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
2787{
2788        struct drm_i915_private *dev_priv = ring->dev->dev_private;
2789        u32 cmd, ipehr, head;
2790        u64 offset = 0;
2791        int i, backwards;
2792
2793        /*
2794         * This function does not support execlist mode - any attempt to
2795         * proceed further into this function will result in a kernel panic
2796         * when dereferencing ring->buffer, which is not set up in execlist
2797         * mode.
2798         *
2799         * The correct way of doing it would be to derive the currently
2800         * executing ring buffer from the current context, which is derived
2801         * from the currently running request. Unfortunately, to get the
2802         * current request we would have to grab the struct_mutex before doing
2803         * anything else, which would be ill-advised since some other thread
2804         * might have grabbed it already and managed to hang itself, causing
2805         * the hang checker to deadlock.
2806         *
2807         * Therefore, this function does not support execlist mode in its
2808         * current form. Just return NULL and move on.
2809         */
2810        if (ring->buffer == NULL)
2811                return NULL;
2812
2813        ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2814        if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2815                return NULL;
2816
2817        /*
2818         * HEAD is likely pointing to the dword after the actual command,
2819         * so scan backwards until we find the MBOX. But limit it to just 3
2820         * or 4 dwords depending on the semaphore wait command size.
2821         * Note that we don't care about ACTHD here since that might
2822         * point at at batch, and semaphores are always emitted into the
2823         * ringbuffer itself.
2824         */
2825        head = I915_READ_HEAD(ring) & HEAD_ADDR;
2826        backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
2827
2828        for (i = backwards; i; --i) {
2829                /*
2830                 * Be paranoid and presume the hw has gone off into the wild -
2831                 * our ring is smaller than what the hardware (and hence
2832                 * HEAD_ADDR) allows. Also handles wrap-around.
2833                 */
2834                head &= ring->buffer->size - 1;
2835
2836                /* This here seems to blow up */
2837                cmd = ioread32(ring->buffer->virtual_start + head);
2838                if (cmd == ipehr)
2839                        break;
2840
2841                head -= 4;
2842        }
2843
2844        if (!i)
2845                return NULL;
2846
2847        *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
2848        if (INTEL_INFO(ring->dev)->gen >= 8) {
2849                offset = ioread32(ring->buffer->virtual_start + head + 12);
2850                offset <<= 32;
2851                offset = ioread32(ring->buffer->virtual_start + head + 8);
2852        }
2853        return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
2854}
2855
2856static int semaphore_passed(struct intel_engine_cs *ring)
2857{
2858        struct drm_i915_private *dev_priv = ring->dev->dev_private;
2859        struct intel_engine_cs *signaller;
2860        u32 seqno;
2861
2862        ring->hangcheck.deadlock++;
2863
2864        signaller = semaphore_waits_for(ring, &seqno);
2865        if (signaller == NULL)
2866                return -1;
2867
2868        /* Prevent pathological recursion due to driver bugs */
2869        if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
2870                return -1;
2871
2872        if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2873                return 1;
2874
2875        /* cursory check for an unkickable deadlock */
2876        if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2877            semaphore_passed(signaller) < 0)
2878                return -1;
2879
2880        return 0;
2881}
2882
2883static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2884{
2885        struct intel_engine_cs *ring;
2886        int i;
2887
2888        for_each_ring(ring, dev_priv, i)
2889                ring->hangcheck.deadlock = 0;
2890}
2891
2892static enum intel_ring_hangcheck_action
2893ring_stuck(struct intel_engine_cs *ring, u64 acthd)
2894{
2895        struct drm_device *dev = ring->dev;
2896        struct drm_i915_private *dev_priv = dev->dev_private;
2897        u32 tmp;
2898
2899        if (acthd != ring->hangcheck.acthd) {
2900                if (acthd > ring->hangcheck.max_acthd) {
2901                        ring->hangcheck.max_acthd = acthd;
2902                        return HANGCHECK_ACTIVE;
2903                }
2904
2905                return HANGCHECK_ACTIVE_LOOP;
2906        }
2907
2908        if (IS_GEN2(dev))
2909                return HANGCHECK_HUNG;
2910
2911        /* Is the chip hanging on a WAIT_FOR_EVENT?
2912         * If so we can simply poke the RB_WAIT bit
2913         * and break the hang. This should work on
2914         * all but the second generation chipsets.
2915         */
2916        tmp = I915_READ_CTL(ring);
2917        if (tmp & RING_WAIT) {
2918                i915_handle_error(dev, false,
2919                                  "Kicking stuck wait on %s",
2920                                  ring->name);
2921                I915_WRITE_CTL(ring, tmp);
2922                return HANGCHECK_KICK;
2923        }
2924
2925        if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2926                switch (semaphore_passed(ring)) {
2927                default:
2928                        return HANGCHECK_HUNG;
2929                case 1:
2930                        i915_handle_error(dev, false,
2931                                          "Kicking stuck semaphore on %s",
2932                                          ring->name);
2933                        I915_WRITE_CTL(ring, tmp);
2934                        return HANGCHECK_KICK;
2935                case 0:
2936                        return HANGCHECK_WAIT;
2937                }
2938        }
2939
2940        return HANGCHECK_HUNG;
2941}
2942
2943/*
2944 * This is called when the chip hasn't reported back with completed
2945 * batchbuffers in a long time. We keep track per ring seqno progress and
2946 * if there are no progress, hangcheck score for that ring is increased.
2947 * Further, acthd is inspected to see if the ring is stuck. On stuck case
2948 * we kick the ring. If we see no progress on three subsequent calls
2949 * we assume chip is wedged and try to fix it by resetting the chip.
2950 */
2951static void i915_hangcheck_elapsed(struct work_struct *work)
2952{
2953        struct drm_i915_private *dev_priv =
2954                container_of(work, typeof(*dev_priv),
2955                             gpu_error.hangcheck_work.work);
2956        struct drm_device *dev = dev_priv->dev;
2957        struct intel_engine_cs *ring;
2958        int i;
2959        int busy_count = 0, rings_hung = 0;
2960        bool stuck[I915_NUM_RINGS] = { 0 };
2961#define BUSY 1
2962#define KICK 5
2963#define HUNG 20
2964
2965        if (!i915.enable_hangcheck)
2966                return;
2967
2968        for_each_ring(ring, dev_priv, i) {
2969                u64 acthd;
2970                u32 seqno;
2971                bool busy = true;
2972
2973                semaphore_clear_deadlocks(dev_priv);
2974
2975                seqno = ring->get_seqno(ring, false);
2976                acthd = intel_ring_get_active_head(ring);
2977
2978                if (ring->hangcheck.seqno == seqno) {
2979                        if (ring_idle(ring, seqno)) {
2980                                ring->hangcheck.action = HANGCHECK_IDLE;
2981
2982                                if (waitqueue_active(&ring->irq_queue)) {
2983                                        /* Issue a wake-up to catch stuck h/w. */
2984                                        if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2985                                                if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2986                                                        DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2987                                                                  ring->name);
2988                                                else
2989                                                        DRM_INFO("Fake missed irq on %s\n",
2990                                                                 ring->name);
2991                                                wake_up_all(&ring->irq_queue);
2992                                        }
2993                                        /* Safeguard against driver failure */
2994                                        ring->hangcheck.score += BUSY;
2995                                } else
2996                                        busy = false;
2997                        } else {
2998                                /* We always increment the hangcheck score
2999                                 * if the ring is busy and still processing
3000                                 * the same request, so that no single request
3001                                 * can run indefinitely (such as a chain of
3002                                 * batches). The only time we do not increment
3003                                 * the hangcheck score on this ring, if this
3004                                 * ring is in a legitimate wait for another
3005                                 * ring. In that case the waiting ring is a
3006                                 * victim and we want to be sure we catch the
3007                                 * right culprit. Then every time we do kick
3008                                 * the ring, add a small increment to the
3009                                 * score so that we can catch a batch that is
3010                                 * being repeatedly kicked and so responsible
3011                                 * for stalling the machine.
3012                                 */
3013                                ring->hangcheck.action = ring_stuck(ring,
3014                                                                    acthd);
3015
3016                                switch (ring->hangcheck.action) {
3017                                case HANGCHECK_IDLE:
3018                                case HANGCHECK_WAIT:
3019                                case HANGCHECK_ACTIVE:
3020                                        break;
3021                                case HANGCHECK_ACTIVE_LOOP:
3022                                        ring->hangcheck.score += BUSY;
3023                                        break;
3024                                case HANGCHECK_KICK:
3025                                        ring->hangcheck.score += KICK;
3026                                        break;
3027                                case HANGCHECK_HUNG:
3028                                        ring->hangcheck.score += HUNG;
3029                                        stuck[i] = true;
3030                                        break;
3031                                }
3032                        }
3033                } else {
3034                        ring->hangcheck.action = HANGCHECK_ACTIVE;
3035
3036                        /* Gradually reduce the count so that we catch DoS
3037                         * attempts across multiple batches.
3038                         */
3039                        if (ring->hangcheck.score > 0)
3040                                ring->hangcheck.score--;
3041
3042                        ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
3043                }
3044
3045                ring->hangcheck.seqno = seqno;
3046                ring->hangcheck.acthd = acthd;
3047                busy_count += busy;
3048        }
3049
3050        for_each_ring(ring, dev_priv, i) {
3051                if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3052                        DRM_INFO("%s on %s\n",
3053                                 stuck[i] ? "stuck" : "no progress",
3054                                 ring->name);
3055                        rings_hung++;
3056                }
3057        }
3058
3059        if (rings_hung)
3060                return i915_handle_error(dev, true, "Ring hung");
3061
3062        if (busy_count)
3063                /* Reset timer case chip hangs without another request
3064                 * being added */
3065                i915_queue_hangcheck(dev);
3066}
3067
3068void i915_queue_hangcheck(struct drm_device *dev)
3069{
3070        struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3071
3072        if (!i915.enable_hangcheck)
3073                return;
3074
3075        /* Don't continually defer the hangcheck so that it is always run at
3076         * least once after work has been scheduled on any ring. Otherwise,
3077         * we will ignore a hung ring if a second ring is kept busy.
3078         */
3079
3080        queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
3081                           round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
3082}
3083
3084static void ibx_irq_reset(struct drm_device *dev)
3085{
3086        struct drm_i915_private *dev_priv = dev->dev_private;
3087
3088        if (HAS_PCH_NOP(dev))
3089                return;
3090
3091        GEN5_IRQ_RESET(SDE);
3092
3093        if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3094                I915_WRITE(SERR_INT, 0xffffffff);
3095}
3096
3097/*
3098 * SDEIER is also touched by the interrupt handler to work around missed PCH
3099 * interrupts. Hence we can't update it after the interrupt handler is enabled -
3100 * instead we unconditionally enable all PCH interrupt sources here, but then
3101 * only unmask them as needed with SDEIMR.
3102 *
3103 * This function needs to be called before interrupts are enabled.
3104 */
3105static void ibx_irq_pre_postinstall(struct drm_device *dev)
3106{
3107        struct drm_i915_private *dev_priv = dev->dev_private;
3108
3109        if (HAS_PCH_NOP(dev))
3110                return;
3111
3112        WARN_ON(I915_READ(SDEIER) != 0);
3113        I915_WRITE(SDEIER, 0xffffffff);
3114        POSTING_READ(SDEIER);
3115}
3116
3117static void gen5_gt_irq_reset(struct drm_device *dev)
3118{
3119        struct drm_i915_private *dev_priv = dev->dev_private;
3120
3121        GEN5_IRQ_RESET(GT);
3122        if (INTEL_INFO(dev)->gen >= 6)
3123                GEN5_IRQ_RESET(GEN6_PM);
3124}
3125
3126/* drm_dma.h hooks
3127*/
3128static void ironlake_irq_reset(struct drm_device *dev)
3129{
3130        struct drm_i915_private *dev_priv = dev->dev_private;
3131
3132        I915_WRITE(HWSTAM, 0xffffffff);
3133
3134        GEN5_IRQ_RESET(DE);
3135        if (IS_GEN7(dev))
3136                I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3137
3138        gen5_gt_irq_reset(dev);
3139
3140        ibx_irq_reset(dev);
3141}
3142
3143static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3144{
3145        enum pipe pipe;
3146
3147        i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
3148        I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3149
3150        for_each_pipe(dev_priv, pipe)
3151                I915_WRITE(PIPESTAT(pipe), 0xffff);
3152
3153        GEN5_IRQ_RESET(VLV_);
3154}
3155
3156static void valleyview_irq_preinstall(struct drm_device *dev)
3157{
3158        struct drm_i915_private *dev_priv = dev->dev_private;
3159
3160        /* VLV magic */
3161        I915_WRITE(VLV_IMR, 0);
3162        I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
3163        I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
3164        I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
3165
3166        gen5_gt_irq_reset(dev);
3167
3168        I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3169
3170        vlv_display_irq_reset(dev_priv);
3171}
3172
3173static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3174{
3175        GEN8_IRQ_RESET_NDX(GT, 0);
3176        GEN8_IRQ_RESET_NDX(GT, 1);
3177        GEN8_IRQ_RESET_NDX(GT, 2);
3178        GEN8_IRQ_RESET_NDX(GT, 3);
3179}
3180
3181static void gen8_irq_reset(struct drm_device *dev)
3182{
3183        struct drm_i915_private *dev_priv = dev->dev_private;
3184        int pipe;
3185
3186        I915_WRITE(GEN8_MASTER_IRQ, 0);
3187        POSTING_READ(GEN8_MASTER_IRQ);
3188
3189        gen8_gt_irq_reset(dev_priv);
3190
3191        for_each_pipe(dev_priv, pipe)
3192                if (intel_display_power_is_enabled(dev_priv,
3193                                                   POWER_DOMAIN_PIPE(pipe)))
3194                        GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3195
3196        GEN5_IRQ_RESET(GEN8_DE_PORT_);
3197        GEN5_IRQ_RESET(GEN8_DE_MISC_);
3198        GEN5_IRQ_RESET(GEN8_PCU_);
3199
3200        if (HAS_PCH_SPLIT(dev))
3201                ibx_irq_reset(dev);
3202}
3203
3204void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3205                                     unsigned int pipe_mask)
3206{
3207        uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3208
3209        spin_lock_irq(&dev_priv->irq_lock);
3210        if (pipe_mask & 1 << PIPE_A)
3211                GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_A,
3212                                  dev_priv->de_irq_mask[PIPE_A],
3213                                  ~dev_priv->de_irq_mask[PIPE_A] | extra_ier);
3214        if (pipe_mask & 1 << PIPE_B)
3215                GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B,
3216                                  dev_priv->de_irq_mask[PIPE_B],
3217                                  ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
3218        if (pipe_mask & 1 << PIPE_C)
3219                GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C,
3220                                  dev_priv->de_irq_mask[PIPE_C],
3221                                  ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
3222        spin_unlock_irq(&dev_priv->irq_lock);
3223}
3224
3225static void cherryview_irq_preinstall(struct drm_device *dev)
3226{
3227        struct drm_i915_private *dev_priv = dev->dev_private;
3228
3229        I915_WRITE(GEN8_MASTER_IRQ, 0);
3230        POSTING_READ(GEN8_MASTER_IRQ);
3231
3232        gen8_gt_irq_reset(dev_priv);
3233
3234        GEN5_IRQ_RESET(GEN8_PCU_);
3235
3236        I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3237
3238        vlv_display_irq_reset(dev_priv);
3239}
3240
3241static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
3242                                  const u32 hpd[HPD_NUM_PINS])
3243{
3244        struct drm_i915_private *dev_priv = to_i915(dev);
3245        struct intel_encoder *encoder;
3246        u32 enabled_irqs = 0;
3247
3248        for_each_intel_encoder(dev, encoder)
3249                if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
3250                        enabled_irqs |= hpd[encoder->hpd_pin];
3251
3252        return enabled_irqs;
3253}
3254
3255static void ibx_hpd_irq_setup(struct drm_device *dev)
3256{
3257        struct drm_i915_private *dev_priv = dev->dev_private;
3258        u32 hotplug_irqs, hotplug, enabled_irqs;
3259
3260        if (HAS_PCH_IBX(dev)) {
3261                hotplug_irqs = SDE_HOTPLUG_MASK;
3262                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3263        } else {
3264                hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3265                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3266        }
3267
3268        ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3269
3270        /*
3271         * Enable digital hotplug on the PCH, and configure the DP short pulse
3272         * duration to 2ms (which is the minimum in the Display Port spec).
3273         * The pulse duration bits are reserved on LPT+.
3274         */
3275        hotplug = I915_READ(PCH_PORT_HOTPLUG);
3276        hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3277        hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3278        hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3279        hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3280        /*
3281         * When CPU and PCH are on the same package, port A
3282         * HPD must be enabled in both north and south.
3283         */
3284        if (HAS_PCH_LPT_LP(dev))
3285                hotplug |= PORTA_HOTPLUG_ENABLE;
3286        I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3287}
3288
3289static void spt_hpd_irq_setup(struct drm_device *dev)
3290{
3291        struct drm_i915_private *dev_priv = dev->dev_private;
3292        u32 hotplug_irqs, hotplug, enabled_irqs;
3293
3294        hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3295        enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
3296
3297        ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3298
3299        /* Enable digital hotplug on the PCH */
3300        hotplug = I915_READ(PCH_PORT_HOTPLUG);
3301        hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3302                PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3303        I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3304
3305        hotplug = I915_READ(PCH_PORT_HOTPLUG2);
3306        hotplug |= PORTE_HOTPLUG_ENABLE;
3307        I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3308}
3309
3310static void ilk_hpd_irq_setup(struct drm_device *dev)
3311{
3312        struct drm_i915_private *dev_priv = dev->dev_private;
3313        u32 hotplug_irqs, hotplug, enabled_irqs;
3314
3315        if (INTEL_INFO(dev)->gen >= 8) {
3316                hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3317                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
3318
3319                bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3320        } else if (INTEL_INFO(dev)->gen >= 7) {
3321                hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3322                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3323
3324                ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3325        } else {
3326                hotplug_irqs = DE_DP_A_HOTPLUG;
3327                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3328
3329                ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3330        }
3331
3332        /*
3333         * Enable digital hotplug on the CPU, and configure the DP short pulse
3334         * duration to 2ms (which is the minimum in the Display Port spec)
3335         * The pulse duration bits are reserved on HSW+.
3336         */
3337        hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
3338        hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
3339        hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
3340        I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
3341
3342        ibx_hpd_irq_setup(dev);
3343}
3344
3345static void bxt_hpd_irq_setup(struct drm_device *dev)
3346{
3347        struct drm_i915_private *dev_priv = dev->dev_private;
3348        u32 hotplug_irqs, hotplug, enabled_irqs;
3349
3350        enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
3351        hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3352
3353        bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3354
3355        hotplug = I915_READ(PCH_PORT_HOTPLUG);
3356        hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
3357                PORTA_HOTPLUG_ENABLE;
3358        I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3359}
3360
3361static void ibx_irq_postinstall(struct drm_device *dev)
3362{
3363        struct drm_i915_private *dev_priv = dev->dev_private;
3364        u32 mask;
3365
3366        if (HAS_PCH_NOP(dev))
3367                return;
3368
3369        if (HAS_PCH_IBX(dev))
3370                mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3371        else
3372                mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3373
3374        gen5_assert_iir_is_zero(dev_priv, SDEIIR);
3375        I915_WRITE(SDEIMR, ~mask);
3376}
3377
3378static void gen5_gt_irq_postinstall(struct drm_device *dev)
3379{
3380        struct drm_i915_private *dev_priv = dev->dev_private;
3381        u32 pm_irqs, gt_irqs;
3382
3383        pm_irqs = gt_irqs = 0;
3384
3385        dev_priv->gt_irq_mask = ~0;
3386        if (HAS_L3_DPF(dev)) {
3387                /* L3 parity interrupt is always unmasked. */
3388                dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3389                gt_irqs |= GT_PARITY_ERROR(dev);
3390        }
3391
3392        gt_irqs |= GT_RENDER_USER_INTERRUPT;
3393        if (IS_GEN5(dev)) {
3394                gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3395                           ILK_BSD_USER_INTERRUPT;
3396        } else {
3397                gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3398        }
3399
3400        GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3401
3402        if (INTEL_INFO(dev)->gen >= 6) {
3403                /*
3404                 * RPS interrupts will get enabled/disabled on demand when RPS
3405                 * itself is enabled/disabled.
3406                 */
3407                if (HAS_VEBOX(dev))
3408                        pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3409
3410                dev_priv->pm_irq_mask = 0xffffffff;
3411                GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3412        }
3413}
3414
3415static int ironlake_irq_postinstall(struct drm_device *dev)
3416{
3417        struct drm_i915_private *dev_priv = dev->dev_private;
3418        u32 display_mask, extra_mask;
3419
3420        if (INTEL_INFO(dev)->gen >= 7) {
3421                display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3422                                DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3423                                DE_PLANEB_FLIP_DONE_IVB |
3424                                DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3425                extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3426                              DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
3427                              DE_DP_A_HOTPLUG_IVB);
3428        } else {
3429                display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3430                                DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3431                                DE_AUX_CHANNEL_A |
3432                                DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3433                                DE_POISON);
3434                extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3435                              DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
3436                              DE_DP_A_HOTPLUG);
3437        }
3438
3439        dev_priv->irq_mask = ~display_mask;
3440
3441        I915_WRITE(HWSTAM, 0xeffe);
3442
3443        ibx_irq_pre_postinstall(dev);
3444
3445        GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3446
3447        gen5_gt_irq_postinstall(dev);
3448
3449        ibx_irq_postinstall(dev);
3450
3451        if (IS_IRONLAKE_M(dev)) {
3452                /* Enable PCU event interrupts
3453                 *
3454                 * spinlocking not required here for correctness since interrupt
3455                 * setup is guaranteed to run in single-threaded context. But we
3456                 * need it to make the assert_spin_locked happy. */
3457                spin_lock_irq(&dev_priv->irq_lock);
3458                ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3459                spin_unlock_irq(&dev_priv->irq_lock);
3460        }
3461
3462        return 0;
3463}
3464
3465static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3466{
3467        u32 pipestat_mask;
3468        u32 iir_mask;
3469        enum pipe pipe;
3470
3471        pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3472                        PIPE_FIFO_UNDERRUN_STATUS;
3473
3474        for_each_pipe(dev_priv, pipe)
3475                I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3476        POSTING_READ(PIPESTAT(PIPE_A));
3477
3478        pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3479                        PIPE_CRC_DONE_INTERRUPT_STATUS;
3480
3481        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3482        for_each_pipe(dev_priv, pipe)
3483                      i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3484
3485        iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3486                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3487                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3488        if (IS_CHERRYVIEW(dev_priv))
3489                iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3490        dev_priv->irq_mask &= ~iir_mask;
3491
3492        I915_WRITE(VLV_IIR, iir_mask);
3493        I915_WRITE(VLV_IIR, iir_mask);
3494        I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3495        I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3496        POSTING_READ(VLV_IMR);
3497}
3498
3499static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3500{
3501        u32 pipestat_mask;
3502        u32 iir_mask;
3503        enum pipe pipe;
3504
3505        iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3506                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3507                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3508        if (IS_CHERRYVIEW(dev_priv))
3509                iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3510
3511        dev_priv->irq_mask |= iir_mask;
3512        I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3513        I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3514        I915_WRITE(VLV_IIR, iir_mask);
3515        I915_WRITE(VLV_IIR, iir_mask);
3516        POSTING_READ(VLV_IIR);
3517
3518        pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3519                        PIPE_CRC_DONE_INTERRUPT_STATUS;
3520
3521        i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3522        for_each_pipe(dev_priv, pipe)
3523                i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3524
3525        pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3526                        PIPE_FIFO_UNDERRUN_STATUS;
3527
3528        for_each_pipe(dev_priv, pipe)
3529                I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3530        POSTING_READ(PIPESTAT(PIPE_A));
3531}
3532
3533void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3534{
3535        assert_spin_locked(&dev_priv->irq_lock);
3536
3537        if (dev_priv->display_irqs_enabled)
3538                return;
3539
3540        dev_priv->display_irqs_enabled = true;
3541
3542        if (intel_irqs_enabled(dev_priv))
3543                valleyview_display_irqs_install(dev_priv);
3544}
3545
3546void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3547{
3548        assert_spin_locked(&dev_priv->irq_lock);
3549
3550        if (!dev_priv->display_irqs_enabled)
3551                return;
3552
3553        dev_priv->display_irqs_enabled = false;
3554
3555        if (intel_irqs_enabled(dev_priv))
3556                valleyview_display_irqs_uninstall(dev_priv);
3557}
3558
3559static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
3560{
3561        dev_priv->irq_mask = ~0;
3562
3563        i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3564        POSTING_READ(PORT_HOTPLUG_EN);
3565
3566        I915_WRITE(VLV_IIR, 0xffffffff);
3567        I915_WRITE(VLV_IIR, 0xffffffff);
3568        I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3569        I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3570        POSTING_READ(VLV_IMR);
3571
3572        /* Interrupt setup is already guaranteed to be single-threaded, this is
3573         * just to make the assert_spin_locked check happy. */
3574        spin_lock_irq(&dev_priv->irq_lock);
3575        if (dev_priv->display_irqs_enabled)
3576                valleyview_display_irqs_install(dev_priv);
3577        spin_unlock_irq(&dev_priv->irq_lock);
3578}
3579
3580static int valleyview_irq_postinstall(struct drm_device *dev)
3581{
3582        struct drm_i915_private *dev_priv = dev->dev_private;
3583
3584        vlv_display_irq_postinstall(dev_priv);
3585
3586        gen5_gt_irq_postinstall(dev);
3587
3588        /* ack & enable invalid PTE error interrupts */
3589#if 0 /* FIXME: add support to irq handler for checking these bits */
3590        I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3591        I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3592#endif
3593
3594        I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3595
3596        return 0;
3597}
3598
3599static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3600{
3601        /* These are interrupts we'll toggle with the ring mask register */
3602        uint32_t gt_interrupts[] = {
3603                GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3604                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3605                        GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3606                        GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3607                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3608                GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3609                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3610                        GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3611                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3612                0,
3613                GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3614                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3615                };
3616
3617        dev_priv->pm_irq_mask = 0xffffffff;
3618        GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3619        GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3620        /*
3621         * RPS interrupts will get enabled/disabled on demand when RPS itself
3622         * is enabled/disabled.
3623         */
3624        GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3625        GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3626}
3627
3628static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3629{
3630        uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3631        uint32_t de_pipe_enables;
3632        u32 de_port_masked = GEN8_AUX_CHANNEL_A;
3633        u32 de_port_enables;
3634        enum pipe pipe;
3635
3636        if (INTEL_INFO(dev_priv)->gen >= 9) {
3637                de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3638                                  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3639                de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3640                                  GEN9_AUX_CHANNEL_D;
3641                if (IS_BROXTON(dev_priv))
3642                        de_port_masked |= BXT_DE_PORT_GMBUS;
3643        } else {
3644                de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3645                                  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3646        }
3647
3648        de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3649                                           GEN8_PIPE_FIFO_UNDERRUN;
3650
3651        de_port_enables = de_port_masked;
3652        if (IS_BROXTON(dev_priv))
3653                de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
3654        else if (IS_BROADWELL(dev_priv))
3655                de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
3656
3657        dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3658        dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3659        dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3660
3661        for_each_pipe(dev_priv, pipe)
3662                if (intel_display_power_is_enabled(dev_priv,
3663                                POWER_DOMAIN_PIPE(pipe)))
3664                        GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3665                                          dev_priv->de_irq_mask[pipe],
3666                                          de_pipe_enables);
3667
3668        GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3669}
3670
3671static int gen8_irq_postinstall(struct drm_device *dev)
3672{
3673        struct drm_i915_private *dev_priv = dev->dev_private;
3674
3675        if (HAS_PCH_SPLIT(dev))
3676                ibx_irq_pre_postinstall(dev);
3677
3678        gen8_gt_irq_postinstall(dev_priv);
3679        gen8_de_irq_postinstall(dev_priv);
3680
3681        if (HAS_PCH_SPLIT(dev))
3682                ibx_irq_postinstall(dev);
3683
3684        I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3685        POSTING_READ(GEN8_MASTER_IRQ);
3686
3687        return 0;
3688}
3689
3690static int cherryview_irq_postinstall(struct drm_device *dev)
3691{
3692        struct drm_i915_private *dev_priv = dev->dev_private;
3693
3694        vlv_display_irq_postinstall(dev_priv);
3695
3696        gen8_gt_irq_postinstall(dev_priv);
3697
3698        I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3699        POSTING_READ(GEN8_MASTER_IRQ);
3700
3701        return 0;
3702}
3703
3704static void gen8_irq_uninstall(struct drm_device *dev)
3705{
3706        struct drm_i915_private *dev_priv = dev->dev_private;
3707
3708        if (!dev_priv)
3709                return;
3710
3711        gen8_irq_reset(dev);
3712}
3713
3714static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3715{
3716        /* Interrupt setup is already guaranteed to be single-threaded, this is
3717         * just to make the assert_spin_locked check happy. */
3718        spin_lock_irq(&dev_priv->irq_lock);
3719        if (dev_priv->display_irqs_enabled)
3720                valleyview_display_irqs_uninstall(dev_priv);
3721        spin_unlock_irq(&dev_priv->irq_lock);
3722
3723        vlv_display_irq_reset(dev_priv);
3724
3725        dev_priv->irq_mask = ~0;
3726}
3727
3728static void valleyview_irq_uninstall(struct drm_device *dev)
3729{
3730        struct drm_i915_private *dev_priv = dev->dev_private;
3731
3732        if (!dev_priv)
3733                return;
3734
3735        I915_WRITE(VLV_MASTER_IER, 0);
3736
3737        gen5_gt_irq_reset(dev);
3738
3739        I915_WRITE(HWSTAM, 0xffffffff);
3740
3741        vlv_display_irq_uninstall(dev_priv);
3742}
3743
3744static void cherryview_irq_uninstall(struct drm_device *dev)
3745{
3746        struct drm_i915_private *dev_priv = dev->dev_private;
3747
3748        if (!dev_priv)
3749                return;
3750
3751        I915_WRITE(GEN8_MASTER_IRQ, 0);
3752        POSTING_READ(GEN8_MASTER_IRQ);
3753
3754        gen8_gt_irq_reset(dev_priv);
3755
3756        GEN5_IRQ_RESET(GEN8_PCU_);
3757
3758        vlv_display_irq_uninstall(dev_priv);
3759}
3760
3761static void ironlake_irq_uninstall(struct drm_device *dev)
3762{
3763        struct drm_i915_private *dev_priv = dev->dev_private;
3764
3765        if (!dev_priv)
3766                return;
3767
3768        ironlake_irq_reset(dev);
3769}
3770
3771static void i8xx_irq_preinstall(struct drm_device * dev)
3772{
3773        struct drm_i915_private *dev_priv = dev->dev_private;
3774        int pipe;
3775
3776        for_each_pipe(dev_priv, pipe)
3777                I915_WRITE(PIPESTAT(pipe), 0);
3778        I915_WRITE16(IMR, 0xffff);
3779        I915_WRITE16(IER, 0x0);
3780        POSTING_READ16(IER);
3781}
3782
3783static int i8xx_irq_postinstall(struct drm_device *dev)
3784{
3785        struct drm_i915_private *dev_priv = dev->dev_private;
3786
3787        I915_WRITE16(EMR,
3788                     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3789
3790        /* Unmask the interrupts that we always want on. */
3791        dev_priv->irq_mask =
3792                ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3793                  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3794                  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3795                  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3796        I915_WRITE16(IMR, dev_priv->irq_mask);
3797
3798        I915_WRITE16(IER,
3799                     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3800                     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3801                     I915_USER_INTERRUPT);
3802        POSTING_READ16(IER);
3803
3804        /* Interrupt setup is already guaranteed to be single-threaded, this is
3805         * just to make the assert_spin_locked check happy. */
3806        spin_lock_irq(&dev_priv->irq_lock);
3807        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3808        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3809        spin_unlock_irq(&dev_priv->irq_lock);
3810
3811        return 0;
3812}
3813
3814/*
3815 * Returns true when a page flip has completed.
3816 */
3817static bool i8xx_handle_vblank(struct drm_device *dev,
3818                               int plane, int pipe, u32 iir)
3819{
3820        struct drm_i915_private *dev_priv = dev->dev_private;
3821        u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3822
3823        if (!intel_pipe_handle_vblank(dev, pipe))
3824                return false;
3825
3826        if ((iir & flip_pending) == 0)
3827                goto check_page_flip;
3828
3829        /* We detect FlipDone by looking for the change in PendingFlip from '1'
3830         * to '0' on the following vblank, i.e. IIR has the Pendingflip
3831         * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3832         * the flip is completed (no longer pending). Since this doesn't raise
3833         * an interrupt per se, we watch for the change at vblank.
3834         */
3835        if (I915_READ16(ISR) & flip_pending)
3836                goto check_page_flip;
3837
3838        intel_prepare_page_flip(dev, plane);
3839        intel_finish_page_flip(dev, pipe);
3840        return true;
3841
3842check_page_flip:
3843        intel_check_page_flip(dev, pipe);
3844        return false;
3845}
3846
3847static irqreturn_t i8xx_irq_handler(int irq, void *arg)
3848{
3849        struct drm_device *dev = arg;
3850        struct drm_i915_private *dev_priv = dev->dev_private;
3851        u16 iir, new_iir;
3852        u32 pipe_stats[2];
3853        int pipe;
3854        u16 flip_mask =
3855                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3856                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3857
3858        if (!intel_irqs_enabled(dev_priv))
3859                return IRQ_NONE;
3860
3861        iir = I915_READ16(IIR);
3862        if (iir == 0)
3863                return IRQ_NONE;
3864
3865        while (iir & ~flip_mask) {
3866                /* Can't rely on pipestat interrupt bit in iir as it might
3867                 * have been cleared after the pipestat interrupt was received.
3868                 * It doesn't set the bit in iir again, but it still produces
3869                 * interrupts (for non-MSI).
3870                 */
3871                spin_lock(&dev_priv->irq_lock);
3872                if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3873                        DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3874
3875                for_each_pipe(dev_priv, pipe) {
3876                        int reg = PIPESTAT(pipe);
3877                        pipe_stats[pipe] = I915_READ(reg);
3878
3879                        /*
3880                         * Clear the PIPE*STAT regs before the IIR
3881                         */
3882                        if (pipe_stats[pipe] & 0x8000ffff)
3883                                I915_WRITE(reg, pipe_stats[pipe]);
3884                }
3885                spin_unlock(&dev_priv->irq_lock);
3886
3887                I915_WRITE16(IIR, iir & ~flip_mask);
3888                new_iir = I915_READ16(IIR); /* Flush posted writes */
3889
3890                if (iir & I915_USER_INTERRUPT)
3891                        notify_ring(&dev_priv->ring[RCS]);
3892
3893                for_each_pipe(dev_priv, pipe) {
3894                        int plane = pipe;
3895                        if (HAS_FBC(dev))
3896                                plane = !plane;
3897
3898                        if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3899                            i8xx_handle_vblank(dev, plane, pipe, iir))
3900                                flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3901
3902                        if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3903                                i9xx_pipe_crc_irq_handler(dev, pipe);
3904
3905                        if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3906                                intel_cpu_fifo_underrun_irq_handler(dev_priv,
3907                                                                    pipe);
3908                }
3909
3910                iir = new_iir;
3911        }
3912
3913        return IRQ_HANDLED;
3914}
3915
3916static void i8xx_irq_uninstall(struct drm_device * dev)
3917{
3918        struct drm_i915_private *dev_priv = dev->dev_private;
3919        int pipe;
3920
3921        for_each_pipe(dev_priv, pipe) {
3922                /* Clear enable bits; then clear status bits */
3923                I915_WRITE(PIPESTAT(pipe), 0);
3924                I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3925        }
3926        I915_WRITE16(IMR, 0xffff);
3927        I915_WRITE16(IER, 0x0);
3928        I915_WRITE16(IIR, I915_READ16(IIR));
3929}
3930
3931static void i915_irq_preinstall(struct drm_device * dev)
3932{
3933        struct drm_i915_private *dev_priv = dev->dev_private;
3934        int pipe;
3935
3936        if (I915_HAS_HOTPLUG(dev)) {
3937                i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3938                I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3939        }
3940
3941        I915_WRITE16(HWSTAM, 0xeffe);
3942        for_each_pipe(dev_priv, pipe)
3943                I915_WRITE(PIPESTAT(pipe), 0);
3944        I915_WRITE(IMR, 0xffffffff);
3945        I915_WRITE(IER, 0x0);
3946        POSTING_READ(IER);
3947}
3948
3949static int i915_irq_postinstall(struct drm_device *dev)
3950{
3951        struct drm_i915_private *dev_priv = dev->dev_private;
3952        u32 enable_mask;
3953
3954        I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3955
3956        /* Unmask the interrupts that we always want on. */
3957        dev_priv->irq_mask =
3958                ~(I915_ASLE_INTERRUPT |
3959                  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3960                  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3961                  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3962                  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3963
3964        enable_mask =
3965                I915_ASLE_INTERRUPT |
3966                I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3967                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3968                I915_USER_INTERRUPT;
3969
3970        if (I915_HAS_HOTPLUG(dev)) {
3971                i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3972                POSTING_READ(PORT_HOTPLUG_EN);
3973
3974                /* Enable in IER... */
3975                enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3976                /* and unmask in IMR */
3977                dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3978        }
3979
3980        I915_WRITE(IMR, dev_priv->irq_mask);
3981        I915_WRITE(IER, enable_mask);
3982        POSTING_READ(IER);
3983
3984        i915_enable_asle_pipestat(dev);
3985
3986        /* Interrupt setup is already guaranteed to be single-threaded, this is
3987         * just to make the assert_spin_locked check happy. */
3988        spin_lock_irq(&dev_priv->irq_lock);
3989        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3990        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3991        spin_unlock_irq(&dev_priv->irq_lock);
3992
3993        return 0;
3994}
3995
3996/*
3997 * Returns true when a page flip has completed.
3998 */
3999static bool i915_handle_vblank(struct drm_device *dev,
4000                               int plane, int pipe, u32 iir)
4001{
4002        struct drm_i915_private *dev_priv = dev->dev_private;
4003        u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
4004
4005        if (!intel_pipe_handle_vblank(dev, pipe))
4006                return false;
4007
4008        if ((iir & flip_pending) == 0)
4009                goto check_page_flip;
4010
4011        /* We detect FlipDone by looking for the change in PendingFlip from '1'
4012         * to '0' on the following vblank, i.e. IIR has the Pendingflip
4013         * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
4014         * the flip is completed (no longer pending). Since this doesn't raise
4015         * an interrupt per se, we watch for the change at vblank.
4016         */
4017        if (I915_READ(ISR) & flip_pending)
4018                goto check_page_flip;
4019
4020        intel_prepare_page_flip(dev, plane);
4021        intel_finish_page_flip(dev, pipe);
4022        return true;
4023
4024check_page_flip:
4025        intel_check_page_flip(dev, pipe);
4026        return false;
4027}
4028
4029static irqreturn_t i915_irq_handler(int irq, void *arg)
4030{
4031        struct drm_device *dev = arg;
4032        struct drm_i915_private *dev_priv = dev->dev_private;
4033        u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4034        u32 flip_mask =
4035                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4036                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4037        int pipe, ret = IRQ_NONE;
4038
4039        if (!intel_irqs_enabled(dev_priv))
4040                return IRQ_NONE;
4041
4042        iir = I915_READ(IIR);
4043        do {
4044                bool irq_received = (iir & ~flip_mask) != 0;
4045                bool blc_event = false;
4046
4047                /* Can't rely on pipestat interrupt bit in iir as it might
4048                 * have been cleared after the pipestat interrupt was received.
4049                 * It doesn't set the bit in iir again, but it still produces
4050                 * interrupts (for non-MSI).
4051                 */
4052                spin_lock(&dev_priv->irq_lock);
4053                if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4054                        DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4055
4056                for_each_pipe(dev_priv, pipe) {
4057                        int reg = PIPESTAT(pipe);
4058                        pipe_stats[pipe] = I915_READ(reg);
4059
4060                        /* Clear the PIPE*STAT regs before the IIR */
4061                        if (pipe_stats[pipe] & 0x8000ffff) {
4062                                I915_WRITE(reg, pipe_stats[pipe]);
4063                                irq_received = true;
4064                        }
4065                }
4066                spin_unlock(&dev_priv->irq_lock);
4067
4068                if (!irq_received)
4069                        break;
4070
4071                /* Consume port.  Then clear IIR or we'll miss events */
4072                if (I915_HAS_HOTPLUG(dev) &&
4073                    iir & I915_DISPLAY_PORT_INTERRUPT)
4074                        i9xx_hpd_irq_handler(dev);
4075
4076                I915_WRITE(IIR, iir & ~flip_mask);
4077                new_iir = I915_READ(IIR); /* Flush posted writes */
4078
4079                if (iir & I915_USER_INTERRUPT)
4080                        notify_ring(&dev_priv->ring[RCS]);
4081
4082                for_each_pipe(dev_priv, pipe) {
4083                        int plane = pipe;
4084                        if (HAS_FBC(dev))
4085                                plane = !plane;
4086
4087                        if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4088                            i915_handle_vblank(dev, plane, pipe, iir))
4089                                flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4090
4091                        if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4092                                blc_event = true;
4093
4094                        if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4095                                i9xx_pipe_crc_irq_handler(dev, pipe);
4096
4097                        if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4098                                intel_cpu_fifo_underrun_irq_handler(dev_priv,
4099                                                                    pipe);
4100                }
4101
4102                if (blc_event || (iir & I915_ASLE_INTERRUPT))
4103                        intel_opregion_asle_intr(dev);
4104
4105                /* With MSI, interrupts are only generated when iir
4106                 * transitions from zero to nonzero.  If another bit got
4107                 * set while we were handling the existing iir bits, then
4108                 * we would never get another interrupt.
4109                 *
4110                 * This is fine on non-MSI as well, as if we hit this path
4111                 * we avoid exiting the interrupt handler only to generate
4112                 * another one.
4113                 *
4114                 * Note that for MSI this could cause a stray interrupt report
4115                 * if an interrupt landed in the time between writing IIR and
4116                 * the posting read.  This should be rare enough to never
4117                 * trigger the 99% of 100,000 interrupts test for disabling
4118                 * stray interrupts.
4119                 */
4120                ret = IRQ_HANDLED;
4121                iir = new_iir;
4122        } while (iir & ~flip_mask);
4123
4124        return ret;
4125}
4126
4127static void i915_irq_uninstall(struct drm_device * dev)
4128{
4129        struct drm_i915_private *dev_priv = dev->dev_private;
4130        int pipe;
4131
4132        if (I915_HAS_HOTPLUG(dev)) {
4133                i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4134                I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4135        }
4136
4137        I915_WRITE16(HWSTAM, 0xffff);
4138        for_each_pipe(dev_priv, pipe) {
4139                /* Clear enable bits; then clear status bits */
4140                I915_WRITE(PIPESTAT(pipe), 0);
4141                I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
4142        }
4143        I915_WRITE(IMR, 0xffffffff);
4144        I915_WRITE(IER, 0x0);
4145
4146        I915_WRITE(IIR, I915_READ(IIR));
4147}
4148
4149static void i965_irq_preinstall(struct drm_device * dev)
4150{
4151        struct drm_i915_private *dev_priv = dev->dev_private;
4152        int pipe;
4153
4154        i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4155        I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4156
4157        I915_WRITE(HWSTAM, 0xeffe);
4158        for_each_pipe(dev_priv, pipe)
4159                I915_WRITE(PIPESTAT(pipe), 0);
4160        I915_WRITE(IMR, 0xffffffff);
4161        I915_WRITE(IER, 0x0);
4162        POSTING_READ(IER);
4163}
4164
4165static int i965_irq_postinstall(struct drm_device *dev)
4166{
4167        struct drm_i915_private *dev_priv = dev->dev_private;
4168        u32 enable_mask;
4169        u32 error_mask;
4170
4171        /* Unmask the interrupts that we always want on. */
4172        dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4173                               I915_DISPLAY_PORT_INTERRUPT |
4174                               I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4175                               I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4176                               I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4177                               I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4178                               I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4179
4180        enable_mask = ~dev_priv->irq_mask;
4181        enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4182                         I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4183        enable_mask |= I915_USER_INTERRUPT;
4184
4185        if (IS_G4X(dev))
4186                enable_mask |= I915_BSD_USER_INTERRUPT;
4187
4188        /* Interrupt setup is already guaranteed to be single-threaded, this is
4189         * just to make the assert_spin_locked check happy. */
4190        spin_lock_irq(&dev_priv->irq_lock);
4191        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4192        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4193        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4194        spin_unlock_irq(&dev_priv->irq_lock);
4195
4196        /*
4197         * Enable some error detection, note the instruction error mask
4198         * bit is reserved, so we leave it masked.
4199         */
4200        if (IS_G4X(dev)) {
4201                error_mask = ~(GM45_ERROR_PAGE_TABLE |
4202                               GM45_ERROR_MEM_PRIV |
4203                               GM45_ERROR_CP_PRIV |
4204                               I915_ERROR_MEMORY_REFRESH);
4205        } else {
4206                error_mask = ~(I915_ERROR_PAGE_TABLE |
4207                               I915_ERROR_MEMORY_REFRESH);
4208        }
4209        I915_WRITE(EMR, error_mask);
4210
4211        I915_WRITE(IMR, dev_priv->irq_mask);
4212        I915_WRITE(IER, enable_mask);
4213        POSTING_READ(IER);
4214
4215        i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4216        POSTING_READ(PORT_HOTPLUG_EN);
4217
4218        i915_enable_asle_pipestat(dev);
4219
4220        return 0;
4221}
4222
4223static void i915_hpd_irq_setup(struct drm_device *dev)
4224{
4225        struct drm_i915_private *dev_priv = dev->dev_private;
4226        u32 hotplug_en;
4227
4228        assert_spin_locked(&dev_priv->irq_lock);
4229
4230        /* Note HDMI and DP share hotplug bits */
4231        /* enable bits are the same for all generations */
4232        hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4233        /* Programming the CRT detection parameters tends
4234           to generate a spurious hotplug event about three
4235           seconds later.  So just do it once.
4236        */
4237        if (IS_G4X(dev))
4238                hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4239        hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4240
4241        /* Ignore TV since it's buggy */
4242        i915_hotplug_interrupt_update_locked(dev_priv,
4243                                             HOTPLUG_INT_EN_MASK |
4244                                             CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
4245                                             CRT_HOTPLUG_ACTIVATION_PERIOD_64,
4246                                             hotplug_en);
4247}
4248
4249static irqreturn_t i965_irq_handler(int irq, void *arg)
4250{
4251        struct drm_device *dev = arg;
4252        struct drm_i915_private *dev_priv = dev->dev_private;
4253        u32 iir, new_iir;
4254        u32 pipe_stats[I915_MAX_PIPES];
4255        int ret = IRQ_NONE, pipe;
4256        u32 flip_mask =
4257                I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4258                I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4259
4260        if (!intel_irqs_enabled(dev_priv))
4261                return IRQ_NONE;
4262
4263        iir = I915_READ(IIR);
4264
4265        for (;;) {
4266                bool irq_received = (iir & ~flip_mask) != 0;
4267                bool blc_event = false;
4268
4269                /* Can't rely on pipestat interrupt bit in iir as it might
4270                 * have been cleared after the pipestat interrupt was received.
4271                 * It doesn't set the bit in iir again, but it still produces
4272                 * interrupts (for non-MSI).
4273                 */
4274                spin_lock(&dev_priv->irq_lock);
4275                if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4276                        DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4277
4278                for_each_pipe(dev_priv, pipe) {
4279                        int reg = PIPESTAT(pipe);
4280                        pipe_stats[pipe] = I915_READ(reg);
4281
4282                        /*
4283                         * Clear the PIPE*STAT regs before the IIR
4284                         */
4285                        if (pipe_stats[pipe] & 0x8000ffff) {
4286                                I915_WRITE(reg, pipe_stats[pipe]);
4287                                irq_received = true;
4288                        }
4289                }
4290                spin_unlock(&dev_priv->irq_lock);
4291
4292                if (!irq_received)
4293                        break;
4294
4295                ret = IRQ_HANDLED;
4296
4297                /* Consume port.  Then clear IIR or we'll miss events */
4298                if (iir & I915_DISPLAY_PORT_INTERRUPT)
4299                        i9xx_hpd_irq_handler(dev);
4300
4301                I915_WRITE(IIR, iir & ~flip_mask);
4302                new_iir = I915_READ(IIR); /* Flush posted writes */
4303
4304                if (iir & I915_USER_INTERRUPT)
4305                        notify_ring(&dev_priv->ring[RCS]);
4306                if (iir & I915_BSD_USER_INTERRUPT)
4307                        notify_ring(&dev_priv->ring[VCS]);
4308
4309                for_each_pipe(dev_priv, pipe) {
4310                        if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4311                            i915_handle_vblank(dev, pipe, pipe, iir))
4312                                flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4313
4314                        if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4315                                blc_event = true;
4316
4317                        if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4318                                i9xx_pipe_crc_irq_handler(dev, pipe);
4319
4320                        if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4321                                intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4322                }
4323
4324                if (blc_event || (iir & I915_ASLE_INTERRUPT))
4325                        intel_opregion_asle_intr(dev);
4326
4327                if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4328                        gmbus_irq_handler(dev);
4329
4330                /* With MSI, interrupts are only generated when iir
4331                 * transitions from zero to nonzero.  If another bit got
4332                 * set while we were handling the existing iir bits, then
4333                 * we would never get another interrupt.
4334                 *
4335                 * This is fine on non-MSI as well, as if we hit this path
4336                 * we avoid exiting the interrupt handler only to generate
4337                 * another one.
4338                 *
4339                 * Note that for MSI this could cause a stray interrupt report
4340                 * if an interrupt landed in the time between writing IIR and
4341                 * the posting read.  This should be rare enough to never
4342                 * trigger the 99% of 100,000 interrupts test for disabling
4343                 * stray interrupts.
4344                 */
4345                iir = new_iir;
4346        }
4347
4348        return ret;
4349}
4350
4351static void i965_irq_uninstall(struct drm_device * dev)
4352{
4353        struct drm_i915_private *dev_priv = dev->dev_private;
4354        int pipe;
4355
4356        if (!dev_priv)
4357                return;
4358
4359        i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4360        I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4361
4362        I915_WRITE(HWSTAM, 0xffffffff);
4363        for_each_pipe(dev_priv, pipe)
4364                I915_WRITE(PIPESTAT(pipe), 0);
4365        I915_WRITE(IMR, 0xffffffff);
4366        I915_WRITE(IER, 0x0);
4367
4368        for_each_pipe(dev_priv, pipe)
4369                I915_WRITE(PIPESTAT(pipe),
4370                           I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4371        I915_WRITE(IIR, I915_READ(IIR));
4372}
4373
4374/**
4375 * intel_irq_init - initializes irq support
4376 * @dev_priv: i915 device instance
4377 *
4378 * This function initializes all the irq support including work items, timers
4379 * and all the vtables. It does not setup the interrupt itself though.
4380 */
4381void intel_irq_init(struct drm_i915_private *dev_priv)
4382{
4383        struct drm_device *dev = dev_priv->dev;
4384
4385        intel_hpd_init_work(dev_priv);
4386
4387        INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4388        INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4389
4390        /* Let's track the enabled rps events */
4391        if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
4392                /* WaGsvRC0ResidencyMethod:vlv */
4393                dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4394        else
4395                dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4396
4397        INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
4398                          i915_hangcheck_elapsed);
4399
4400        pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4401
4402        if (IS_GEN2(dev_priv)) {
4403                dev->max_vblank_count = 0;
4404                dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4405        } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4406                dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4407                dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4408        } else {
4409                dev->driver->get_vblank_counter = i915_get_vblank_counter;
4410                dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4411        }
4412
4413        /*
4414         * Opt out of the vblank disable timer on everything except gen2.
4415         * Gen2 doesn't have a hardware frame counter and so depends on
4416         * vblank interrupts to produce sane vblank seuquence numbers.
4417         */
4418        if (!IS_GEN2(dev_priv))
4419                dev->vblank_disable_immediate = true;
4420
4421        dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4422        dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4423
4424        if (IS_CHERRYVIEW(dev_priv)) {
4425                dev->driver->irq_handler = cherryview_irq_handler;
4426                dev->driver->irq_preinstall = cherryview_irq_preinstall;
4427                dev->driver->irq_postinstall = cherryview_irq_postinstall;
4428                dev->driver->irq_uninstall = cherryview_irq_uninstall;
4429                dev->driver->enable_vblank = valleyview_enable_vblank;
4430                dev->driver->disable_vblank = valleyview_disable_vblank;
4431                dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4432        } else if (IS_VALLEYVIEW(dev_priv)) {
4433                dev->driver->irq_handler = valleyview_irq_handler;
4434                dev->driver->irq_preinstall = valleyview_irq_preinstall;
4435                dev->driver->irq_postinstall = valleyview_irq_postinstall;
4436                dev->driver->irq_uninstall = valleyview_irq_uninstall;
4437                dev->driver->enable_vblank = valleyview_enable_vblank;
4438                dev->driver->disable_vblank = valleyview_disable_vblank;
4439                dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4440        } else if (INTEL_INFO(dev_priv)->gen >= 8) {
4441                dev->driver->irq_handler = gen8_irq_handler;
4442                dev->driver->irq_preinstall = gen8_irq_reset;
4443                dev->driver->irq_postinstall = gen8_irq_postinstall;
4444                dev->driver->irq_uninstall = gen8_irq_uninstall;
4445                dev->driver->enable_vblank = gen8_enable_vblank;
4446                dev->driver->disable_vblank = gen8_disable_vblank;
4447                if (IS_BROXTON(dev))
4448                        dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4449                else if (HAS_PCH_SPT(dev))
4450                        dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4451                else
4452                        dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4453        } else if (HAS_PCH_SPLIT(dev)) {
4454                dev->driver->irq_handler = ironlake_irq_handler;
4455                dev->driver->irq_preinstall = ironlake_irq_reset;
4456                dev->driver->irq_postinstall = ironlake_irq_postinstall;
4457                dev->driver->irq_uninstall = ironlake_irq_uninstall;
4458                dev->driver->enable_vblank = ironlake_enable_vblank;
4459                dev->driver->disable_vblank = ironlake_disable_vblank;
4460                dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4461        } else {
4462                if (INTEL_INFO(dev_priv)->gen == 2) {
4463                        dev->driver->irq_preinstall = i8xx_irq_preinstall;
4464                        dev->driver->irq_postinstall = i8xx_irq_postinstall;
4465                        dev->driver->irq_handler = i8xx_irq_handler;
4466                        dev->driver->irq_uninstall = i8xx_irq_uninstall;
4467                } else if (INTEL_INFO(dev_priv)->gen == 3) {
4468                        dev->driver->irq_preinstall = i915_irq_preinstall;
4469                        dev->driver->irq_postinstall = i915_irq_postinstall;
4470                        dev->driver->irq_uninstall = i915_irq_uninstall;
4471                        dev->driver->irq_handler = i915_irq_handler;
4472                } else {
4473                        dev->driver->irq_preinstall = i965_irq_preinstall;
4474                        dev->driver->irq_postinstall = i965_irq_postinstall;
4475                        dev->driver->irq_uninstall = i965_irq_uninstall;
4476                        dev->driver->irq_handler = i965_irq_handler;
4477                }
4478                if (I915_HAS_HOTPLUG(dev_priv))
4479                        dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4480                dev->driver->enable_vblank = i915_enable_vblank;
4481                dev->driver->disable_vblank = i915_disable_vblank;
4482        }
4483}
4484
4485/**
4486 * intel_irq_install - enables the hardware interrupt
4487 * @dev_priv: i915 device instance
4488 *
4489 * This function enables the hardware interrupt handling, but leaves the hotplug
4490 * handling still disabled. It is called after intel_irq_init().
4491 *
4492 * In the driver load and resume code we need working interrupts in a few places
4493 * but don't want to deal with the hassle of concurrent probe and hotplug
4494 * workers. Hence the split into this two-stage approach.
4495 */
4496int intel_irq_install(struct drm_i915_private *dev_priv)
4497{
4498        /*
4499         * We enable some interrupt sources in our postinstall hooks, so mark
4500         * interrupts as enabled _before_ actually enabling them to avoid
4501         * special cases in our ordering checks.
4502         */
4503        dev_priv->pm.irqs_enabled = true;
4504
4505        return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4506}
4507
4508/**
4509 * intel_irq_uninstall - finilizes all irq handling
4510 * @dev_priv: i915 device instance
4511 *
4512 * This stops interrupt and hotplug handling and unregisters and frees all
4513 * resources acquired in the init functions.
4514 */
4515void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4516{
4517        drm_irq_uninstall(dev_priv->dev);
4518        intel_hpd_cancel_work(dev_priv);
4519        dev_priv->pm.irqs_enabled = false;
4520}
4521
4522/**
4523 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4524 * @dev_priv: i915 device instance
4525 *
4526 * This function is used to disable interrupts at runtime, both in the runtime
4527 * pm and the system suspend/resume code.
4528 */
4529void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4530{
4531        dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4532        dev_priv->pm.irqs_enabled = false;
4533        synchronize_irq(dev_priv->dev->irq);
4534}
4535
4536/**
4537 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4538 * @dev_priv: i915 device instance
4539 *
4540 * This function is used to enable interrupts at runtime, both in the runtime
4541 * pm and the system suspend/resume code.
4542 */
4543void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4544{
4545        dev_priv->pm.irqs_enabled = true;
4546        dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4547        dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4548}
Note: See TracBrowser for help on using the repository browser.