source: src/linux/universal/linux-4.9/drivers/gpu/drm/i915/i915_drv.c @ 31884

Last change on this file since 31884 was 31884, checked in by brainslayer, 7 days ago

update kernels

File size: 73.3 KB
Line 
1/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
2 */
3/*
4 *
5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 */
29
30#include <linux/acpi.h>
31#include <linux/device.h>
32#include <linux/oom.h>
33#include <linux/module.h>
34#include <linux/pci.h>
35#include <linux/pm.h>
36#include <linux/pm_runtime.h>
37#include <linux/pnp.h>
38#include <linux/slab.h>
39#include <linux/vgaarb.h>
40#include <linux/vga_switcheroo.h>
41#include <linux/vt.h>
42#include <acpi/video.h>
43
44#include <drm/drmP.h>
45#include <drm/drm_crtc_helper.h>
46#include <drm/i915_drm.h>
47
48#include "i915_drv.h"
49#include "i915_trace.h"
50#include "i915_vgpu.h"
51#include "intel_drv.h"
52
53static struct drm_driver driver;
54
55static unsigned int i915_load_fail_count;
56
57bool __i915_inject_load_failure(const char *func, int line)
58{
59        if (i915_load_fail_count >= i915.inject_load_failure)
60                return false;
61
62        if (++i915_load_fail_count == i915.inject_load_failure) {
63                DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
64                         i915.inject_load_failure, func, line);
65                return true;
66        }
67
68        return false;
69}
70
71#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
72#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
73                    "providing the dmesg log by booting with drm.debug=0xf"
74
75void
76__i915_printk(struct drm_i915_private *dev_priv, const char *level,
77              const char *fmt, ...)
78{
79        static bool shown_bug_once;
80        struct device *kdev = dev_priv->drm.dev;
81        bool is_error = level[1] <= KERN_ERR[1];
82        bool is_debug = level[1] == KERN_DEBUG[1];
83        struct va_format vaf;
84        va_list args;
85
86        if (is_debug && !(drm_debug & DRM_UT_DRIVER))
87                return;
88
89        va_start(args, fmt);
90
91        vaf.fmt = fmt;
92        vaf.va = &args;
93
94        dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV",
95                   __builtin_return_address(0), &vaf);
96
97        if (is_error && !shown_bug_once) {
98                dev_notice(kdev, "%s", FDO_BUG_MSG);
99                shown_bug_once = true;
100        }
101
102        va_end(args);
103}
104
105static bool i915_error_injected(struct drm_i915_private *dev_priv)
106{
107        return i915.inject_load_failure &&
108               i915_load_fail_count == i915.inject_load_failure;
109}
110
111#define i915_load_error(dev_priv, fmt, ...)                                  \
112        __i915_printk(dev_priv,                                              \
113                      i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
114                      fmt, ##__VA_ARGS__)
115
116
117static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
118{
119        enum intel_pch ret = PCH_NOP;
120
121        /*
122         * In a virtualized passthrough environment we can be in a
123         * setup where the ISA bridge is not able to be passed through.
124         * In this case, a south bridge can be emulated and we have to
125         * make an educated guess as to which PCH is really there.
126         */
127
128        if (IS_GEN5(dev)) {
129                ret = PCH_IBX;
130                DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
131        } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
132                ret = PCH_CPT;
133                DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
134        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
135                ret = PCH_LPT;
136                DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
137        } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
138                ret = PCH_SPT;
139                DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
140        }
141
142        return ret;
143}
144
145static void intel_detect_pch(struct drm_device *dev)
146{
147        struct drm_i915_private *dev_priv = to_i915(dev);
148        struct pci_dev *pch = NULL;
149
150        /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
151         * (which really amounts to a PCH but no South Display).
152         */
153        if (INTEL_INFO(dev)->num_pipes == 0) {
154                dev_priv->pch_type = PCH_NOP;
155                return;
156        }
157
158        /*
159         * The reason to probe ISA bridge instead of Dev31:Fun0 is to
160         * make graphics device passthrough work easy for VMM, that only
161         * need to expose ISA bridge to let driver know the real hardware
162         * underneath. This is a requirement from virtualization team.
163         *
164         * In some virtualized environments (e.g. XEN), there is irrelevant
165         * ISA bridge in the system. To work reliably, we should scan trhough
166         * all the ISA bridge devices and check for the first match, instead
167         * of only checking the first one.
168         */
169        while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
170                if (pch->vendor == PCI_VENDOR_ID_INTEL) {
171                        unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
172                        dev_priv->pch_id = id;
173
174                        if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
175                                dev_priv->pch_type = PCH_IBX;
176                                DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
177                                WARN_ON(!IS_GEN5(dev));
178                        } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
179                                dev_priv->pch_type = PCH_CPT;
180                                DRM_DEBUG_KMS("Found CougarPoint PCH\n");
181                                WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
182                        } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
183                                /* PantherPoint is CPT compatible */
184                                dev_priv->pch_type = PCH_CPT;
185                                DRM_DEBUG_KMS("Found PantherPoint PCH\n");
186                                WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
187                        } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
188                                dev_priv->pch_type = PCH_LPT;
189                                DRM_DEBUG_KMS("Found LynxPoint PCH\n");
190                                WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
191                                WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
192                        } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
193                                dev_priv->pch_type = PCH_LPT;
194                                DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
195                                WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
196                                WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
197                        } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
198                                dev_priv->pch_type = PCH_SPT;
199                                DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
200                                WARN_ON(!IS_SKYLAKE(dev) &&
201                                        !IS_KABYLAKE(dev));
202                        } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
203                                dev_priv->pch_type = PCH_SPT;
204                                DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
205                                WARN_ON(!IS_SKYLAKE(dev) &&
206                                        !IS_KABYLAKE(dev));
207                        } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
208                                dev_priv->pch_type = PCH_KBP;
209                                DRM_DEBUG_KMS("Found KabyPoint PCH\n");
210                                WARN_ON(!IS_KABYLAKE(dev));
211                        } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
212                                   (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
213                                   ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
214                                    pch->subsystem_vendor ==
215                                            PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
216                                    pch->subsystem_device ==
217                                            PCI_SUBDEVICE_ID_QEMU)) {
218                                dev_priv->pch_type = intel_virt_detect_pch(dev);
219                        } else
220                                continue;
221
222                        break;
223                }
224        }
225        if (!pch)
226                DRM_DEBUG_KMS("No PCH found.\n");
227
228        pci_dev_put(pch);
229}
230
231static int i915_getparam(struct drm_device *dev, void *data,
232                         struct drm_file *file_priv)
233{
234        struct drm_i915_private *dev_priv = to_i915(dev);
235        struct pci_dev *pdev = dev_priv->drm.pdev;
236        drm_i915_getparam_t *param = data;
237        int value;
238
239        switch (param->param) {
240        case I915_PARAM_IRQ_ACTIVE:
241        case I915_PARAM_ALLOW_BATCHBUFFER:
242        case I915_PARAM_LAST_DISPATCH:
243        case I915_PARAM_HAS_EXEC_CONSTANTS:
244                /* Reject all old ums/dri params. */
245                return -ENODEV;
246        case I915_PARAM_CHIPSET_ID:
247                value = pdev->device;
248                break;
249        case I915_PARAM_REVISION:
250                value = pdev->revision;
251                break;
252        case I915_PARAM_NUM_FENCES_AVAIL:
253                value = dev_priv->num_fence_regs;
254                break;
255        case I915_PARAM_HAS_OVERLAY:
256                value = dev_priv->overlay ? 1 : 0;
257                break;
258        case I915_PARAM_HAS_BSD:
259                value = intel_engine_initialized(&dev_priv->engine[VCS]);
260                break;
261        case I915_PARAM_HAS_BLT:
262                value = intel_engine_initialized(&dev_priv->engine[BCS]);
263                break;
264        case I915_PARAM_HAS_VEBOX:
265                value = intel_engine_initialized(&dev_priv->engine[VECS]);
266                break;
267        case I915_PARAM_HAS_BSD2:
268                value = intel_engine_initialized(&dev_priv->engine[VCS2]);
269                break;
270        case I915_PARAM_HAS_LLC:
271                value = HAS_LLC(dev_priv);
272                break;
273        case I915_PARAM_HAS_WT:
274                value = HAS_WT(dev_priv);
275                break;
276        case I915_PARAM_HAS_ALIASING_PPGTT:
277                value = USES_PPGTT(dev_priv);
278                break;
279        case I915_PARAM_HAS_SEMAPHORES:
280                value = i915.semaphores;
281                break;
282        case I915_PARAM_HAS_SECURE_BATCHES:
283                value = capable(CAP_SYS_ADMIN);
284                break;
285        case I915_PARAM_CMD_PARSER_VERSION:
286                value = i915_cmd_parser_get_version(dev_priv);
287                break;
288        case I915_PARAM_SUBSLICE_TOTAL:
289                value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu);
290                if (!value)
291                        return -ENODEV;
292                break;
293        case I915_PARAM_EU_TOTAL:
294                value = INTEL_INFO(dev_priv)->sseu.eu_total;
295                if (!value)
296                        return -ENODEV;
297                break;
298        case I915_PARAM_HAS_GPU_RESET:
299                value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
300                break;
301        case I915_PARAM_HAS_RESOURCE_STREAMER:
302                value = HAS_RESOURCE_STREAMER(dev_priv);
303                break;
304        case I915_PARAM_HAS_POOLED_EU:
305                value = HAS_POOLED_EU(dev_priv);
306                break;
307        case I915_PARAM_MIN_EU_IN_POOL:
308                value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool;
309                break;
310        case I915_PARAM_MMAP_GTT_VERSION:
311                /* Though we've started our numbering from 1, and so class all
312                 * earlier versions as 0, in effect their value is undefined as
313                 * the ioctl will report EINVAL for the unknown param!
314                 */
315                value = i915_gem_mmap_gtt_version();
316                break;
317        case I915_PARAM_MMAP_VERSION:
318                /* Remember to bump this if the version changes! */
319        case I915_PARAM_HAS_GEM:
320        case I915_PARAM_HAS_PAGEFLIPPING:
321        case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */
322        case I915_PARAM_HAS_RELAXED_FENCING:
323        case I915_PARAM_HAS_COHERENT_RINGS:
324        case I915_PARAM_HAS_RELAXED_DELTA:
325        case I915_PARAM_HAS_GEN7_SOL_RESET:
326        case I915_PARAM_HAS_WAIT_TIMEOUT:
327        case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
328        case I915_PARAM_HAS_PINNED_BATCHES:
329        case I915_PARAM_HAS_EXEC_NO_RELOC:
330        case I915_PARAM_HAS_EXEC_HANDLE_LUT:
331        case I915_PARAM_HAS_COHERENT_PHYS_GTT:
332        case I915_PARAM_HAS_EXEC_SOFTPIN:
333                /* For the time being all of these are always true;
334                 * if some supported hardware does not have one of these
335                 * features this value needs to be provided from
336                 * INTEL_INFO(), a feature macro, or similar.
337                 */
338                value = 1;
339                break;
340        default:
341                DRM_DEBUG("Unknown parameter %d\n", param->param);
342                return -EINVAL;
343        }
344
345        if (put_user(value, param->value))
346                return -EFAULT;
347
348        return 0;
349}
350
351static int i915_get_bridge_dev(struct drm_device *dev)
352{
353        struct drm_i915_private *dev_priv = to_i915(dev);
354
355        dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
356        if (!dev_priv->bridge_dev) {
357                DRM_ERROR("bridge device not found\n");
358                return -1;
359        }
360        return 0;
361}
362
363/* Allocate space for the MCH regs if needed, return nonzero on error */
364static int
365intel_alloc_mchbar_resource(struct drm_device *dev)
366{
367        struct drm_i915_private *dev_priv = to_i915(dev);
368        int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
369        u32 temp_lo, temp_hi = 0;
370        u64 mchbar_addr;
371        int ret;
372
373        if (INTEL_INFO(dev)->gen >= 4)
374                pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
375        pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
376        mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
377
378        /* If ACPI doesn't have it, assume we need to allocate it ourselves */
379#ifdef CONFIG_PNP
380        if (mchbar_addr &&
381            pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
382                return 0;
383#endif
384
385        /* Get some space for it */
386        dev_priv->mch_res.name = "i915 MCHBAR";
387        dev_priv->mch_res.flags = IORESOURCE_MEM;
388        ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
389                                     &dev_priv->mch_res,
390                                     MCHBAR_SIZE, MCHBAR_SIZE,
391                                     PCIBIOS_MIN_MEM,
392                                     0, pcibios_align_resource,
393                                     dev_priv->bridge_dev);
394        if (ret) {
395                DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
396                dev_priv->mch_res.start = 0;
397                return ret;
398        }
399
400        if (INTEL_INFO(dev)->gen >= 4)
401                pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
402                                       upper_32_bits(dev_priv->mch_res.start));
403
404        pci_write_config_dword(dev_priv->bridge_dev, reg,
405                               lower_32_bits(dev_priv->mch_res.start));
406        return 0;
407}
408
409/* Setup MCHBAR if possible, return true if we should disable it again */
410static void
411intel_setup_mchbar(struct drm_device *dev)
412{
413        struct drm_i915_private *dev_priv = to_i915(dev);
414        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
415        u32 temp;
416        bool enabled;
417
418        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
419                return;
420
421        dev_priv->mchbar_need_disable = false;
422
423        if (IS_I915G(dev) || IS_I915GM(dev)) {
424                pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
425                enabled = !!(temp & DEVEN_MCHBAR_EN);
426        } else {
427                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
428                enabled = temp & 1;
429        }
430
431        /* If it's already enabled, don't have to do anything */
432        if (enabled)
433                return;
434
435        if (intel_alloc_mchbar_resource(dev))
436                return;
437
438        dev_priv->mchbar_need_disable = true;
439
440        /* Space is allocated or reserved, so enable it. */
441        if (IS_I915G(dev) || IS_I915GM(dev)) {
442                pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
443                                       temp | DEVEN_MCHBAR_EN);
444        } else {
445                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
446                pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
447        }
448}
449
450static void
451intel_teardown_mchbar(struct drm_device *dev)
452{
453        struct drm_i915_private *dev_priv = to_i915(dev);
454        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
455
456        if (dev_priv->mchbar_need_disable) {
457                if (IS_I915G(dev) || IS_I915GM(dev)) {
458                        u32 deven_val;
459
460                        pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
461                                              &deven_val);
462                        deven_val &= ~DEVEN_MCHBAR_EN;
463                        pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
464                                               deven_val);
465                } else {
466                        u32 mchbar_val;
467
468                        pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
469                                              &mchbar_val);
470                        mchbar_val &= ~1;
471                        pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
472                                               mchbar_val);
473                }
474        }
475
476        if (dev_priv->mch_res.start)
477                release_resource(&dev_priv->mch_res);
478}
479
480/* true = enable decode, false = disable decoder */
481static unsigned int i915_vga_set_decode(void *cookie, bool state)
482{
483        struct drm_device *dev = cookie;
484
485        intel_modeset_vga_set_state(dev, state);
486        if (state)
487                return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
488                       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
489        else
490                return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
491}
492
493static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
494{
495        struct drm_device *dev = pci_get_drvdata(pdev);
496        pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
497
498        if (state == VGA_SWITCHEROO_ON) {
499                pr_info("switched on\n");
500                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
501                /* i915 resume handler doesn't set to D0 */
502                pci_set_power_state(pdev, PCI_D0);
503                i915_resume_switcheroo(dev);
504                dev->switch_power_state = DRM_SWITCH_POWER_ON;
505        } else {
506                pr_info("switched off\n");
507                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
508                i915_suspend_switcheroo(dev, pmm);
509                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
510        }
511}
512
513static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
514{
515        struct drm_device *dev = pci_get_drvdata(pdev);
516
517        /*
518         * FIXME: open_count is protected by drm_global_mutex but that would lead to
519         * locking inversion with the driver load path. And the access here is
520         * completely racy anyway. So don't bother with locking for now.
521         */
522        return dev->open_count == 0;
523}
524
525static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
526        .set_gpu_state = i915_switcheroo_set_state,
527        .reprobe = NULL,
528        .can_switch = i915_switcheroo_can_switch,
529};
530
531static void i915_gem_fini(struct drm_device *dev)
532{
533        struct drm_i915_private *dev_priv = to_i915(dev);
534
535        /*
536         * Neither the BIOS, ourselves or any other kernel
537         * expects the system to be in execlists mode on startup,
538         * so we need to reset the GPU back to legacy mode. And the only
539         * known way to disable logical contexts is through a GPU reset.
540         *
541         * So in order to leave the system in a known default configuration,
542         * always reset the GPU upon unload. Afterwards we then clean up the
543         * GEM state tracking, flushing off the requests and leaving the
544         * system in a known idle state.
545         *
546         * Note that is of the upmost importance that the GPU is idle and
547         * all stray writes are flushed *before* we dismantle the backing
548         * storage for the pinned objects.
549         *
550         * However, since we are uncertain that reseting the GPU on older
551         * machines is a good idea, we don't - just in case it leaves the
552         * machine in an unusable condition.
553         */
554        if (HAS_HW_CONTEXTS(dev)) {
555                int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
556                WARN_ON(reset && reset != -ENODEV);
557        }
558
559        mutex_lock(&dev->struct_mutex);
560        i915_gem_cleanup_engines(dev);
561        i915_gem_context_fini(dev);
562        mutex_unlock(&dev->struct_mutex);
563
564        WARN_ON(!list_empty(&to_i915(dev)->context_list));
565}
566
567static int i915_load_modeset_init(struct drm_device *dev)
568{
569        struct drm_i915_private *dev_priv = to_i915(dev);
570        struct pci_dev *pdev = dev_priv->drm.pdev;
571        int ret;
572
573        if (i915_inject_load_failure())
574                return -ENODEV;
575
576        ret = intel_bios_init(dev_priv);
577        if (ret)
578                DRM_INFO("failed to find VBIOS tables\n");
579
580        /* If we have > 1 VGA cards, then we need to arbitrate access
581         * to the common VGA resources.
582         *
583         * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
584         * then we do not take part in VGA arbitration and the
585         * vga_client_register() fails with -ENODEV.
586         */
587        ret = vga_client_register(pdev, dev, NULL, i915_vga_set_decode);
588        if (ret && ret != -ENODEV)
589                goto out;
590
591        intel_register_dsm_handler();
592
593        ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false);
594        if (ret)
595                goto cleanup_vga_client;
596
597        /* must happen before intel_power_domains_init_hw() on VLV/CHV */
598        intel_update_rawclk(dev_priv);
599
600        intel_power_domains_init_hw(dev_priv, false);
601
602        intel_csr_ucode_init(dev_priv);
603
604        ret = intel_irq_install(dev_priv);
605        if (ret)
606                goto cleanup_csr;
607
608        intel_setup_gmbus(dev);
609
610        /* Important: The output setup functions called by modeset_init need
611         * working irqs for e.g. gmbus and dp aux transfers. */
612        intel_modeset_init(dev);
613
614        intel_guc_init(dev);
615
616        ret = i915_gem_init(dev);
617        if (ret)
618                goto cleanup_irq;
619
620        intel_modeset_gem_init(dev);
621
622        if (INTEL_INFO(dev)->num_pipes == 0)
623                return 0;
624
625        ret = intel_fbdev_init(dev);
626        if (ret)
627                goto cleanup_gem;
628
629        /* Only enable hotplug handling once the fbdev is fully set up. */
630        intel_hpd_init(dev_priv);
631
632        drm_kms_helper_poll_init(dev);
633
634        return 0;
635
636cleanup_gem:
637        i915_gem_fini(dev);
638cleanup_irq:
639        intel_guc_fini(dev);
640        drm_irq_uninstall(dev);
641        intel_teardown_gmbus(dev);
642cleanup_csr:
643        intel_csr_ucode_fini(dev_priv);
644        intel_power_domains_fini(dev_priv);
645        vga_switcheroo_unregister_client(pdev);
646cleanup_vga_client:
647        vga_client_register(pdev, NULL, NULL, NULL);
648out:
649        return ret;
650}
651
652#if IS_ENABLED(CONFIG_FB)
653static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
654{
655        struct apertures_struct *ap;
656        struct pci_dev *pdev = dev_priv->drm.pdev;
657        struct i915_ggtt *ggtt = &dev_priv->ggtt;
658        bool primary;
659        int ret;
660
661        ap = alloc_apertures(1);
662        if (!ap)
663                return -ENOMEM;
664
665        ap->ranges[0].base = ggtt->mappable_base;
666        ap->ranges[0].size = ggtt->mappable_end;
667
668        primary =
669                pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
670
671        ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
672
673        kfree(ap);
674
675        return ret;
676}
677#else
678static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
679{
680        return 0;
681}
682#endif
683
684#if !defined(CONFIG_VGA_CONSOLE)
685static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
686{
687        return 0;
688}
689#elif !defined(CONFIG_DUMMY_CONSOLE)
690static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
691{
692        return -ENODEV;
693}
694#else
695static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
696{
697        int ret = 0;
698
699        DRM_INFO("Replacing VGA console driver\n");
700
701        console_lock();
702        if (con_is_bound(&vga_con))
703                ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
704        if (ret == 0) {
705                ret = do_unregister_con_driver(&vga_con);
706
707                /* Ignore "already unregistered". */
708                if (ret == -ENODEV)
709                        ret = 0;
710        }
711        console_unlock();
712
713        return ret;
714}
715#endif
716
717static void intel_init_dpio(struct drm_i915_private *dev_priv)
718{
719        /*
720         * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
721         * CHV x1 PHY (DP/HDMI D)
722         * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
723         */
724        if (IS_CHERRYVIEW(dev_priv)) {
725                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
726                DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
727        } else if (IS_VALLEYVIEW(dev_priv)) {
728                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
729        }
730}
731
732static int i915_workqueues_init(struct drm_i915_private *dev_priv)
733{
734        /*
735         * The i915 workqueue is primarily used for batched retirement of
736         * requests (and thus managing bo) once the task has been completed
737         * by the GPU. i915_gem_retire_requests() is called directly when we
738         * need high-priority retirement, such as waiting for an explicit
739         * bo.
740         *
741         * It is also used for periodic low-priority events, such as
742         * idle-timers and recording error state.
743         *
744         * All tasks on the workqueue are expected to acquire the dev mutex
745         * so there is no point in running more than one instance of the
746         * workqueue at any time.  Use an ordered one.
747         */
748        dev_priv->wq = alloc_ordered_workqueue("i915", 0);
749        if (dev_priv->wq == NULL)
750                goto out_err;
751
752        dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
753        if (dev_priv->hotplug.dp_wq == NULL)
754                goto out_free_wq;
755
756        return 0;
757
758out_free_wq:
759        destroy_workqueue(dev_priv->wq);
760out_err:
761        DRM_ERROR("Failed to allocate workqueues.\n");
762
763        return -ENOMEM;
764}
765
766static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
767{
768        destroy_workqueue(dev_priv->hotplug.dp_wq);
769        destroy_workqueue(dev_priv->wq);
770}
771
772/**
773 * i915_driver_init_early - setup state not requiring device access
774 * @dev_priv: device private
775 *
776 * Initialize everything that is a "SW-only" state, that is state not
777 * requiring accessing the device or exposing the driver via kernel internal
778 * or userspace interfaces. Example steps belonging here: lock initialization,
779 * system memory allocation, setting up device specific attributes and
780 * function hooks not requiring accessing the device.
781 */
782static int i915_driver_init_early(struct drm_i915_private *dev_priv,
783                                  const struct pci_device_id *ent)
784{
785        const struct intel_device_info *match_info =
786                (struct intel_device_info *)ent->driver_data;
787        struct intel_device_info *device_info;
788        int ret = 0;
789
790        if (i915_inject_load_failure())
791                return -ENODEV;
792
793        /* Setup the write-once "constant" device info */
794        device_info = mkwrite_device_info(dev_priv);
795        memcpy(device_info, match_info, sizeof(*device_info));
796        device_info->device_id = dev_priv->drm.pdev->device;
797
798        BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
799        device_info->gen_mask = BIT(device_info->gen - 1);
800
801        spin_lock_init(&dev_priv->irq_lock);
802        spin_lock_init(&dev_priv->gpu_error.lock);
803        mutex_init(&dev_priv->backlight_lock);
804        spin_lock_init(&dev_priv->uncore.lock);
805        spin_lock_init(&dev_priv->mm.object_stat_lock);
806        spin_lock_init(&dev_priv->mmio_flip_lock);
807        mutex_init(&dev_priv->sb_lock);
808        mutex_init(&dev_priv->modeset_restore_lock);
809        mutex_init(&dev_priv->av_mutex);
810        mutex_init(&dev_priv->wm.wm_mutex);
811        mutex_init(&dev_priv->pps_mutex);
812
813        i915_memcpy_init_early(dev_priv);
814
815        ret = i915_workqueues_init(dev_priv);
816        if (ret < 0)
817                return ret;
818
819        ret = intel_gvt_init(dev_priv);
820        if (ret < 0)
821                goto err_workqueues;
822
823        /* This must be called before any calls to HAS_PCH_* */
824        intel_detect_pch(&dev_priv->drm);
825
826        intel_pm_setup(&dev_priv->drm);
827        intel_init_dpio(dev_priv);
828        intel_power_domains_init(dev_priv);
829        intel_irq_init(dev_priv);
830        intel_init_display_hooks(dev_priv);
831        intel_init_clock_gating_hooks(dev_priv);
832        intel_init_audio_hooks(dev_priv);
833        i915_gem_load_init(&dev_priv->drm);
834
835        intel_display_crc_init(dev_priv);
836
837        intel_device_info_dump(dev_priv);
838
839        /* Not all pre-production machines fall into this category, only the
840         * very first ones. Almost everything should work, except for maybe
841         * suspend/resume. And we don't implement workarounds that affect only
842         * pre-production machines. */
843        if (IS_HSW_EARLY_SDV(dev_priv))
844                DRM_INFO("This is an early pre-production Haswell machine. "
845                         "It may not be fully functional.\n");
846
847        return 0;
848
849err_workqueues:
850        i915_workqueues_cleanup(dev_priv);
851        return ret;
852}
853
854/**
855 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
856 * @dev_priv: device private
857 */
858static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
859{
860        i915_gem_load_cleanup(&dev_priv->drm);
861        i915_workqueues_cleanup(dev_priv);
862}
863
864static int i915_mmio_setup(struct drm_device *dev)
865{
866        struct drm_i915_private *dev_priv = to_i915(dev);
867        struct pci_dev *pdev = dev_priv->drm.pdev;
868        int mmio_bar;
869        int mmio_size;
870
871        mmio_bar = IS_GEN2(dev) ? 1 : 0;
872        /*
873         * Before gen4, the registers and the GTT are behind different BARs.
874         * However, from gen4 onwards, the registers and the GTT are shared
875         * in the same BAR, so we want to restrict this ioremap from
876         * clobbering the GTT which we want ioremap_wc instead. Fortunately,
877         * the register BAR remains the same size for all the earlier
878         * generations up to Ironlake.
879         */
880        if (INTEL_INFO(dev)->gen < 5)
881                mmio_size = 512 * 1024;
882        else
883                mmio_size = 2 * 1024 * 1024;
884        dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size);
885        if (dev_priv->regs == NULL) {
886                DRM_ERROR("failed to map registers\n");
887
888                return -EIO;
889        }
890
891        /* Try to make sure MCHBAR is enabled before poking at it */
892        intel_setup_mchbar(dev);
893
894        return 0;
895}
896
897static void i915_mmio_cleanup(struct drm_device *dev)
898{
899        struct drm_i915_private *dev_priv = to_i915(dev);
900        struct pci_dev *pdev = dev_priv->drm.pdev;
901
902        intel_teardown_mchbar(dev);
903        pci_iounmap(pdev, dev_priv->regs);
904}
905
906/**
907 * i915_driver_init_mmio - setup device MMIO
908 * @dev_priv: device private
909 *
910 * Setup minimal device state necessary for MMIO accesses later in the
911 * initialization sequence. The setup here should avoid any other device-wide
912 * side effects or exposing the driver via kernel internal or user space
913 * interfaces.
914 */
915static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
916{
917        struct drm_device *dev = &dev_priv->drm;
918        int ret;
919
920        if (i915_inject_load_failure())
921                return -ENODEV;
922
923        if (i915_get_bridge_dev(dev))
924                return -EIO;
925
926        ret = i915_mmio_setup(dev);
927        if (ret < 0)
928                goto put_bridge;
929
930        intel_uncore_init(dev_priv);
931
932        return 0;
933
934put_bridge:
935        pci_dev_put(dev_priv->bridge_dev);
936
937        return ret;
938}
939
940/**
941 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
942 * @dev_priv: device private
943 */
944static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
945{
946        struct drm_device *dev = &dev_priv->drm;
947
948        intel_uncore_fini(dev_priv);
949        i915_mmio_cleanup(dev);
950        pci_dev_put(dev_priv->bridge_dev);
951}
952
953static void intel_sanitize_options(struct drm_i915_private *dev_priv)
954{
955        i915.enable_execlists =
956                intel_sanitize_enable_execlists(dev_priv,
957                                                i915.enable_execlists);
958
959        /*
960         * i915.enable_ppgtt is read-only, so do an early pass to validate the
961         * user's requested state against the hardware/driver capabilities.  We
962         * do this now so that we can print out any log messages once rather
963         * than every time we check intel_enable_ppgtt().
964         */
965        i915.enable_ppgtt =
966                intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
967        DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
968
969        i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
970        DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores));
971}
972
973/**
974 * i915_driver_init_hw - setup state requiring device access
975 * @dev_priv: device private
976 *
977 * Setup state that requires accessing the device, but doesn't require
978 * exposing the driver via kernel internal or userspace interfaces.
979 */
980static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
981{
982        struct pci_dev *pdev = dev_priv->drm.pdev;
983        struct drm_device *dev = &dev_priv->drm;
984        int ret;
985
986        if (i915_inject_load_failure())
987                return -ENODEV;
988
989        intel_device_info_runtime_init(dev_priv);
990
991        intel_sanitize_options(dev_priv);
992
993        ret = i915_ggtt_probe_hw(dev_priv);
994        if (ret)
995                return ret;
996
997        /* WARNING: Apparently we must kick fbdev drivers before vgacon,
998         * otherwise the vga fbdev driver falls over. */
999        ret = i915_kick_out_firmware_fb(dev_priv);
1000        if (ret) {
1001                DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
1002                goto out_ggtt;
1003        }
1004
1005        ret = i915_kick_out_vgacon(dev_priv);
1006        if (ret) {
1007                DRM_ERROR("failed to remove conflicting VGA console\n");
1008                goto out_ggtt;
1009        }
1010
1011        ret = i915_ggtt_init_hw(dev_priv);
1012        if (ret)
1013                return ret;
1014
1015        ret = i915_ggtt_enable_hw(dev_priv);
1016        if (ret) {
1017                DRM_ERROR("failed to enable GGTT\n");
1018                goto out_ggtt;
1019        }
1020
1021        pci_set_master(pdev);
1022
1023        /* overlay on gen2 is broken and can't address above 1G */
1024        if (IS_GEN2(dev)) {
1025                ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
1026                if (ret) {
1027                        DRM_ERROR("failed to set DMA mask\n");
1028
1029                        goto out_ggtt;
1030                }
1031        }
1032
1033        /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1034         * using 32bit addressing, overwriting memory if HWS is located
1035         * above 4GB.
1036         *
1037         * The documentation also mentions an issue with undefined
1038         * behaviour if any general state is accessed within a page above 4GB,
1039         * which also needs to be handled carefully.
1040         */
1041        if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
1042                ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1043
1044                if (ret) {
1045                        DRM_ERROR("failed to set DMA mask\n");
1046
1047                        goto out_ggtt;
1048                }
1049        }
1050
1051        pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
1052                           PM_QOS_DEFAULT_VALUE);
1053
1054        intel_uncore_sanitize(dev_priv);
1055
1056        intel_opregion_setup(dev_priv);
1057
1058        i915_gem_load_init_fences(dev_priv);
1059
1060        /* On the 945G/GM, the chipset reports the MSI capability on the
1061         * integrated graphics even though the support isn't actually there
1062         * according to the published specs.  It doesn't appear to function
1063         * correctly in testing on 945G.
1064         * This may be a side effect of MSI having been made available for PEG
1065         * and the registers being closely associated.
1066         *
1067         * According to chipset errata, on the 965GM, MSI interrupts may
1068         * be lost or delayed, but we use them anyways to avoid
1069         * stuck interrupts on some machines.
1070         */
1071        if (!IS_I945G(dev) && !IS_I945GM(dev)) {
1072                if (pci_enable_msi(pdev) < 0)
1073                        DRM_DEBUG_DRIVER("can't enable MSI");
1074        }
1075
1076        return 0;
1077
1078out_ggtt:
1079        i915_ggtt_cleanup_hw(dev_priv);
1080
1081        return ret;
1082}
1083
1084/**
1085 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
1086 * @dev_priv: device private
1087 */
1088static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
1089{
1090        struct pci_dev *pdev = dev_priv->drm.pdev;
1091
1092        if (pdev->msi_enabled)
1093                pci_disable_msi(pdev);
1094
1095        pm_qos_remove_request(&dev_priv->pm_qos);
1096        i915_ggtt_cleanup_hw(dev_priv);
1097}
1098
1099/**
1100 * i915_driver_register - register the driver with the rest of the system
1101 * @dev_priv: device private
1102 *
1103 * Perform any steps necessary to make the driver available via kernel
1104 * internal or userspace interfaces.
1105 */
1106static void i915_driver_register(struct drm_i915_private *dev_priv)
1107{
1108        struct drm_device *dev = &dev_priv->drm;
1109
1110        i915_gem_shrinker_init(dev_priv);
1111
1112        /*
1113         * Notify a valid surface after modesetting,
1114         * when running inside a VM.
1115         */
1116        if (intel_vgpu_active(dev_priv))
1117                I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
1118
1119        /* Reveal our presence to userspace */
1120        if (drm_dev_register(dev, 0) == 0) {
1121                i915_debugfs_register(dev_priv);
1122                i915_setup_sysfs(dev_priv);
1123        } else
1124                DRM_ERROR("Failed to register driver for userspace access!\n");
1125
1126        if (INTEL_INFO(dev_priv)->num_pipes) {
1127                /* Must be done after probing outputs */
1128                intel_opregion_register(dev_priv);
1129                acpi_video_register();
1130        }
1131
1132        if (IS_GEN5(dev_priv))
1133                intel_gpu_ips_init(dev_priv);
1134
1135        i915_audio_component_init(dev_priv);
1136
1137        /*
1138         * Some ports require correctly set-up hpd registers for detection to
1139         * work properly (leading to ghost connected connector status), e.g. VGA
1140         * on gm45.  Hence we can only set up the initial fbdev config after hpd
1141         * irqs are fully enabled. We do it last so that the async config
1142         * cannot run before the connectors are registered.
1143         */
1144        intel_fbdev_initial_config_async(dev);
1145}
1146
1147/**
1148 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
1149 * @dev_priv: device private
1150 */
1151static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1152{
1153        i915_audio_component_cleanup(dev_priv);
1154
1155        intel_gpu_ips_teardown();
1156        acpi_video_unregister();
1157        intel_opregion_unregister(dev_priv);
1158
1159        i915_teardown_sysfs(dev_priv);
1160        i915_debugfs_unregister(dev_priv);
1161        drm_dev_unregister(&dev_priv->drm);
1162
1163        i915_gem_shrinker_cleanup(dev_priv);
1164}
1165
1166/**
1167 * i915_driver_load - setup chip and create an initial config
1168 * @dev: DRM device
1169 * @flags: startup flags
1170 *
1171 * The driver load routine has to do several things:
1172 *   - drive output discovery via intel_modeset_init()
1173 *   - initialize the memory manager
1174 *   - allocate initial config memory
1175 *   - setup the DRM framebuffer with the allocated memory
1176 */
1177int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
1178{
1179        struct drm_i915_private *dev_priv;
1180        int ret;
1181
1182        if (i915.nuclear_pageflip)
1183                driver.driver_features |= DRIVER_ATOMIC;
1184
1185        ret = -ENOMEM;
1186        dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
1187        if (dev_priv)
1188                ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev);
1189        if (ret) {
1190                dev_printk(KERN_ERR, &pdev->dev,
1191                           "[" DRM_NAME ":%s] allocation failed\n", __func__);
1192                kfree(dev_priv);
1193                return ret;
1194        }
1195
1196        dev_priv->drm.pdev = pdev;
1197        dev_priv->drm.dev_private = dev_priv;
1198
1199        ret = pci_enable_device(pdev);
1200        if (ret)
1201                goto out_free_priv;
1202
1203        pci_set_drvdata(pdev, &dev_priv->drm);
1204
1205        ret = i915_driver_init_early(dev_priv, ent);
1206        if (ret < 0)
1207                goto out_pci_disable;
1208
1209        intel_runtime_pm_get(dev_priv);
1210
1211        ret = i915_driver_init_mmio(dev_priv);
1212        if (ret < 0)
1213                goto out_runtime_pm_put;
1214
1215        ret = i915_driver_init_hw(dev_priv);
1216        if (ret < 0)
1217                goto out_cleanup_mmio;
1218
1219        /*
1220         * TODO: move the vblank init and parts of modeset init steps into one
1221         * of the i915_driver_init_/i915_driver_register functions according
1222         * to the role/effect of the given init step.
1223         */
1224        if (INTEL_INFO(dev_priv)->num_pipes) {
1225                ret = drm_vblank_init(&dev_priv->drm,
1226                                      INTEL_INFO(dev_priv)->num_pipes);
1227                if (ret)
1228                        goto out_cleanup_hw;
1229        }
1230
1231        ret = i915_load_modeset_init(&dev_priv->drm);
1232        if (ret < 0)
1233                goto out_cleanup_vblank;
1234
1235        i915_driver_register(dev_priv);
1236
1237        intel_runtime_pm_enable(dev_priv);
1238
1239        /* Everything is in place, we can now relax! */
1240        DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
1241                 driver.name, driver.major, driver.minor, driver.patchlevel,
1242                 driver.date, pci_name(pdev), dev_priv->drm.primary->index);
1243
1244        intel_runtime_pm_put(dev_priv);
1245
1246        return 0;
1247
1248out_cleanup_vblank:
1249        drm_vblank_cleanup(&dev_priv->drm);
1250out_cleanup_hw:
1251        i915_driver_cleanup_hw(dev_priv);
1252out_cleanup_mmio:
1253        i915_driver_cleanup_mmio(dev_priv);
1254out_runtime_pm_put:
1255        intel_runtime_pm_put(dev_priv);
1256        i915_driver_cleanup_early(dev_priv);
1257out_pci_disable:
1258        pci_disable_device(pdev);
1259out_free_priv:
1260        i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
1261        drm_dev_unref(&dev_priv->drm);
1262        return ret;
1263}
1264
1265void i915_driver_unload(struct drm_device *dev)
1266{
1267        struct drm_i915_private *dev_priv = to_i915(dev);
1268        struct pci_dev *pdev = dev_priv->drm.pdev;
1269
1270        intel_fbdev_fini(dev);
1271
1272        if (i915_gem_suspend(dev))
1273                DRM_ERROR("failed to idle hardware; continuing to unload!\n");
1274
1275        intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
1276
1277        i915_driver_unregister(dev_priv);
1278
1279        drm_vblank_cleanup(dev);
1280
1281        intel_modeset_cleanup(dev);
1282
1283        /*
1284         * free the memory space allocated for the child device
1285         * config parsed from VBT
1286         */
1287        if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
1288                kfree(dev_priv->vbt.child_dev);
1289                dev_priv->vbt.child_dev = NULL;
1290                dev_priv->vbt.child_dev_num = 0;
1291        }
1292        kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
1293        dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
1294        kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
1295        dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
1296
1297        vga_switcheroo_unregister_client(pdev);
1298        vga_client_register(pdev, NULL, NULL, NULL);
1299
1300        intel_csr_ucode_fini(dev_priv);
1301
1302        /* Free error state after interrupts are fully disabled. */
1303        cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
1304        i915_destroy_error_state(dev);
1305
1306        /* Flush any outstanding unpin_work. */
1307        drain_workqueue(dev_priv->wq);
1308
1309        intel_guc_fini(dev);
1310        i915_gem_fini(dev);
1311        intel_fbc_cleanup_cfb(dev_priv);
1312
1313        intel_power_domains_fini(dev_priv);
1314
1315        i915_driver_cleanup_hw(dev_priv);
1316        i915_driver_cleanup_mmio(dev_priv);
1317
1318        intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
1319
1320        i915_driver_cleanup_early(dev_priv);
1321}
1322
1323static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
1324{
1325        int ret;
1326
1327        ret = i915_gem_open(dev, file);
1328        if (ret)
1329                return ret;
1330
1331        return 0;
1332}
1333
1334/**
1335 * i915_driver_lastclose - clean up after all DRM clients have exited
1336 * @dev: DRM device
1337 *
1338 * Take care of cleaning up after all DRM clients have exited.  In the
1339 * mode setting case, we want to restore the kernel's initial mode (just
1340 * in case the last client left us in a bad state).
1341 *
1342 * Additionally, in the non-mode setting case, we'll tear down the GTT
1343 * and DMA structures, since the kernel won't be using them, and clea
1344 * up any GEM state.
1345 */
1346static void i915_driver_lastclose(struct drm_device *dev)
1347{
1348        intel_fbdev_restore_mode(dev);
1349        vga_switcheroo_process_delayed_switch();
1350}
1351
1352static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
1353{
1354        mutex_lock(&dev->struct_mutex);
1355        i915_gem_context_close(dev, file);
1356        i915_gem_release(dev, file);
1357        mutex_unlock(&dev->struct_mutex);
1358}
1359
1360static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1361{
1362        struct drm_i915_file_private *file_priv = file->driver_priv;
1363
1364        kfree(file_priv);
1365}
1366
1367static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
1368{
1369        struct drm_device *dev = &dev_priv->drm;
1370        struct intel_encoder *encoder;
1371
1372        drm_modeset_lock_all(dev);
1373        for_each_intel_encoder(dev, encoder)
1374                if (encoder->suspend)
1375                        encoder->suspend(encoder);
1376        drm_modeset_unlock_all(dev);
1377}
1378
1379static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1380                              bool rpm_resume);
1381static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
1382
1383static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1384{
1385#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1386        if (acpi_target_system_state() < ACPI_STATE_S3)
1387                return true;
1388#endif
1389        return false;
1390}
1391
1392static int i915_drm_suspend(struct drm_device *dev)
1393{
1394        struct drm_i915_private *dev_priv = to_i915(dev);
1395        struct pci_dev *pdev = dev_priv->drm.pdev;
1396        pci_power_t opregion_target_state;
1397        int error;
1398
1399        /* ignore lid events during suspend */
1400        mutex_lock(&dev_priv->modeset_restore_lock);
1401        dev_priv->modeset_restore = MODESET_SUSPENDED;
1402        mutex_unlock(&dev_priv->modeset_restore_lock);
1403
1404        disable_rpm_wakeref_asserts(dev_priv);
1405
1406        /* We do a lot of poking in a lot of registers, make sure they work
1407         * properly. */
1408        intel_display_set_init_power(dev_priv, true);
1409
1410        drm_kms_helper_poll_disable(dev);
1411
1412        pci_save_state(pdev);
1413
1414        error = i915_gem_suspend(dev);
1415        if (error) {
1416                dev_err(&pdev->dev,
1417                        "GEM idle failed, resume might fail\n");
1418                goto out;
1419        }
1420
1421        intel_guc_suspend(dev);
1422
1423        intel_display_suspend(dev);
1424
1425        intel_dp_mst_suspend(dev);
1426
1427        intel_runtime_pm_disable_interrupts(dev_priv);
1428        intel_hpd_cancel_work(dev_priv);
1429
1430        intel_suspend_encoders(dev_priv);
1431
1432        intel_suspend_hw(dev);
1433
1434        i915_gem_suspend_gtt_mappings(dev);
1435
1436        i915_save_state(dev);
1437
1438        opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
1439        intel_opregion_notify_adapter(dev_priv, opregion_target_state);
1440
1441        intel_uncore_forcewake_reset(dev_priv, false);
1442        intel_opregion_unregister(dev_priv);
1443
1444        intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
1445
1446        dev_priv->suspend_count++;
1447
1448        intel_csr_ucode_suspend(dev_priv);
1449
1450out:
1451        enable_rpm_wakeref_asserts(dev_priv);
1452
1453        return error;
1454}
1455
1456static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1457{
1458        struct drm_i915_private *dev_priv = to_i915(dev);
1459        struct pci_dev *pdev = dev_priv->drm.pdev;
1460        bool fw_csr;
1461        int ret;
1462
1463        disable_rpm_wakeref_asserts(dev_priv);
1464
1465        intel_display_set_init_power(dev_priv, false);
1466
1467        fw_csr = !IS_BROXTON(dev_priv) &&
1468                suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
1469        /*
1470         * In case of firmware assisted context save/restore don't manually
1471         * deinit the power domains. This also means the CSR/DMC firmware will
1472         * stay active, it will power down any HW resources as required and
1473         * also enable deeper system power states that would be blocked if the
1474         * firmware was inactive.
1475         */
1476        if (!fw_csr)
1477                intel_power_domains_suspend(dev_priv);
1478
1479        ret = 0;
1480        if (IS_BROXTON(dev_priv))
1481                bxt_enable_dc9(dev_priv);
1482        else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
1483                hsw_enable_pc8(dev_priv);
1484        else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1485                ret = vlv_suspend_complete(dev_priv);
1486
1487        if (ret) {
1488                DRM_ERROR("Suspend complete failed: %d\n", ret);
1489                if (!fw_csr)
1490                        intel_power_domains_init_hw(dev_priv, true);
1491
1492                goto out;
1493        }
1494
1495        pci_disable_device(pdev);
1496        /*
1497         * During hibernation on some platforms the BIOS may try to access
1498         * the device even though it's already in D3 and hang the machine. So
1499         * leave the device in D0 on those platforms and hope the BIOS will
1500         * power down the device properly. The issue was seen on multiple old
1501         * GENs with different BIOS vendors, so having an explicit blacklist
1502         * is inpractical; apply the workaround on everything pre GEN6. The
1503         * platforms where the issue was seen:
1504         * Lenovo Thinkpad X301, X61s, X60, T60, X41
1505         * Fujitsu FSC S7110
1506         * Acer Aspire 1830T
1507         */
1508        if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
1509                pci_set_power_state(pdev, PCI_D3hot);
1510
1511        dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
1512
1513out:
1514        enable_rpm_wakeref_asserts(dev_priv);
1515
1516        return ret;
1517}
1518
1519int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
1520{
1521        int error;
1522
1523        if (!dev) {
1524                DRM_ERROR("dev: %p\n", dev);
1525                DRM_ERROR("DRM not initialized, aborting suspend.\n");
1526                return -ENODEV;
1527        }
1528
1529        if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
1530                         state.event != PM_EVENT_FREEZE))
1531                return -EINVAL;
1532
1533        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1534                return 0;
1535
1536        error = i915_drm_suspend(dev);
1537        if (error)
1538                return error;
1539
1540        return i915_drm_suspend_late(dev, false);
1541}
1542
1543static int i915_drm_resume(struct drm_device *dev)
1544{
1545        struct drm_i915_private *dev_priv = to_i915(dev);
1546        int ret;
1547
1548        disable_rpm_wakeref_asserts(dev_priv);
1549        intel_sanitize_gt_powersave(dev_priv);
1550
1551        ret = i915_ggtt_enable_hw(dev_priv);
1552        if (ret)
1553                DRM_ERROR("failed to re-enable GGTT\n");
1554
1555        intel_csr_ucode_resume(dev_priv);
1556
1557        i915_gem_resume(dev);
1558
1559        i915_restore_state(dev);
1560        intel_pps_unlock_regs_wa(dev_priv);
1561        intel_opregion_setup(dev_priv);
1562
1563        intel_init_pch_refclk(dev);
1564        drm_mode_config_reset(dev);
1565
1566        /*
1567         * Interrupts have to be enabled before any batches are run. If not the
1568         * GPU will hang. i915_gem_init_hw() will initiate batches to
1569         * update/restore the context.
1570         *
1571         * Modeset enabling in intel_modeset_init_hw() also needs working
1572         * interrupts.
1573         */
1574        intel_runtime_pm_enable_interrupts(dev_priv);
1575
1576        mutex_lock(&dev->struct_mutex);
1577        if (i915_gem_init_hw(dev)) {
1578                DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
1579                i915_gem_set_wedged(dev_priv);
1580        }
1581        mutex_unlock(&dev->struct_mutex);
1582
1583        intel_guc_resume(dev);
1584
1585        intel_modeset_init_hw(dev);
1586
1587        spin_lock_irq(&dev_priv->irq_lock);
1588        if (dev_priv->display.hpd_irq_setup)
1589                dev_priv->display.hpd_irq_setup(dev_priv);
1590        spin_unlock_irq(&dev_priv->irq_lock);
1591
1592        intel_dp_mst_resume(dev);
1593
1594        intel_display_resume(dev);
1595
1596        /*
1597         * ... but also need to make sure that hotplug processing
1598         * doesn't cause havoc. Like in the driver load code we don't
1599         * bother with the tiny race here where we might loose hotplug
1600         * notifications.
1601         * */
1602        intel_hpd_init(dev_priv);
1603        /* Config may have changed between suspend and resume */
1604        drm_helper_hpd_irq_event(dev);
1605
1606        intel_opregion_register(dev_priv);
1607
1608        intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
1609
1610        mutex_lock(&dev_priv->modeset_restore_lock);
1611        dev_priv->modeset_restore = MODESET_DONE;
1612        mutex_unlock(&dev_priv->modeset_restore_lock);
1613
1614        intel_opregion_notify_adapter(dev_priv, PCI_D0);
1615
1616        intel_autoenable_gt_powersave(dev_priv);
1617        drm_kms_helper_poll_enable(dev);
1618
1619        enable_rpm_wakeref_asserts(dev_priv);
1620
1621        return 0;
1622}
1623
1624static int i915_drm_resume_early(struct drm_device *dev)
1625{
1626        struct drm_i915_private *dev_priv = to_i915(dev);
1627        struct pci_dev *pdev = dev_priv->drm.pdev;
1628        int ret;
1629
1630        /*
1631         * We have a resume ordering issue with the snd-hda driver also
1632         * requiring our device to be power up. Due to the lack of a
1633         * parent/child relationship we currently solve this with an early
1634         * resume hook.
1635         *
1636         * FIXME: This should be solved with a special hdmi sink device or
1637         * similar so that power domains can be employed.
1638         */
1639
1640        /*
1641         * Note that we need to set the power state explicitly, since we
1642         * powered off the device during freeze and the PCI core won't power
1643         * it back up for us during thaw. Powering off the device during
1644         * freeze is not a hard requirement though, and during the
1645         * suspend/resume phases the PCI core makes sure we get here with the
1646         * device powered on. So in case we change our freeze logic and keep
1647         * the device powered we can also remove the following set power state
1648         * call.
1649         */
1650        ret = pci_set_power_state(pdev, PCI_D0);
1651        if (ret) {
1652                DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
1653                goto out;
1654        }
1655
1656        /*
1657         * Note that pci_enable_device() first enables any parent bridge
1658         * device and only then sets the power state for this device. The
1659         * bridge enabling is a nop though, since bridge devices are resumed
1660         * first. The order of enabling power and enabling the device is
1661         * imposed by the PCI core as described above, so here we preserve the
1662         * same order for the freeze/thaw phases.
1663         *
1664         * TODO: eventually we should remove pci_disable_device() /
1665         * pci_enable_enable_device() from suspend/resume. Due to how they
1666         * depend on the device enable refcount we can't anyway depend on them
1667         * disabling/enabling the device.
1668         */
1669        if (pci_enable_device(pdev)) {
1670                ret = -EIO;
1671                goto out;
1672        }
1673
1674        pci_set_master(pdev);
1675
1676        disable_rpm_wakeref_asserts(dev_priv);
1677
1678        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1679                ret = vlv_resume_prepare(dev_priv, false);
1680        if (ret)
1681                DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
1682                          ret);
1683
1684        intel_uncore_early_sanitize(dev_priv, true);
1685
1686        if (IS_BROXTON(dev_priv)) {
1687                if (!dev_priv->suspended_to_idle)
1688                        gen9_sanitize_dc_state(dev_priv);
1689                bxt_disable_dc9(dev_priv);
1690        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1691                hsw_disable_pc8(dev_priv);
1692        }
1693
1694        intel_uncore_sanitize(dev_priv);
1695
1696        if (IS_BROXTON(dev_priv) ||
1697            !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
1698                intel_power_domains_init_hw(dev_priv, true);
1699
1700        enable_rpm_wakeref_asserts(dev_priv);
1701
1702out:
1703        dev_priv->suspended_to_idle = false;
1704
1705        return ret;
1706}
1707
1708int i915_resume_switcheroo(struct drm_device *dev)
1709{
1710        int ret;
1711
1712        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1713                return 0;
1714
1715        ret = i915_drm_resume_early(dev);
1716        if (ret)
1717                return ret;
1718
1719        return i915_drm_resume(dev);
1720}
1721
1722/**
1723 * i915_reset - reset chip after a hang
1724 * @dev: drm device to reset
1725 *
1726 * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
1727 * on failure.
1728 *
1729 * Caller must hold the struct_mutex.
1730 *
1731 * Procedure is fairly simple:
1732 *   - reset the chip using the reset reg
1733 *   - re-init context state
1734 *   - re-init hardware status page
1735 *   - re-init ring buffer
1736 *   - re-init interrupt state
1737 *   - re-init display
1738 */
1739void i915_reset(struct drm_i915_private *dev_priv)
1740{
1741        struct drm_device *dev = &dev_priv->drm;
1742        struct i915_gpu_error *error = &dev_priv->gpu_error;
1743        int ret;
1744
1745        lockdep_assert_held(&dev->struct_mutex);
1746
1747        if (!test_and_clear_bit(I915_RESET_IN_PROGRESS, &error->flags))
1748                return;
1749
1750        /* Clear any previous failed attempts at recovery. Time to try again. */
1751        __clear_bit(I915_WEDGED, &error->flags);
1752        error->reset_count++;
1753
1754        pr_notice("drm/i915: Resetting chip after gpu hang\n");
1755        ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
1756        if (ret) {
1757                if (ret != -ENODEV)
1758                        DRM_ERROR("Failed to reset chip: %i\n", ret);
1759                else
1760                        DRM_DEBUG_DRIVER("GPU reset disabled\n");
1761                goto error;
1762        }
1763
1764        i915_gem_reset(dev_priv);
1765        intel_overlay_reset(dev_priv);
1766
1767        /* Ok, now get things going again... */
1768
1769        /*
1770         * Everything depends on having the GTT running, so we need to start
1771         * there.  Fortunately we don't need to do this unless we reset the
1772         * chip at a PCI level.
1773         *
1774         * Next we need to restore the context, but we don't use those
1775         * yet either...
1776         *
1777         * Ring buffer needs to be re-initialized in the KMS case, or if X
1778         * was running at the time of the reset (i.e. we weren't VT
1779         * switched away).
1780         */
1781        ret = i915_gem_init_hw(dev);
1782        if (ret) {
1783                DRM_ERROR("Failed hw init on reset %d\n", ret);
1784                goto error;
1785        }
1786
1787wakeup:
1788        wake_up_bit(&error->flags, I915_RESET_IN_PROGRESS);
1789        return;
1790
1791error:
1792        i915_gem_set_wedged(dev_priv);
1793        goto wakeup;
1794}
1795
1796static int i915_pm_suspend(struct device *kdev)
1797{
1798        struct pci_dev *pdev = to_pci_dev(kdev);
1799        struct drm_device *dev = pci_get_drvdata(pdev);
1800
1801        if (!dev) {
1802                dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1803                return -ENODEV;
1804        }
1805
1806        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1807                return 0;
1808
1809        return i915_drm_suspend(dev);
1810}
1811
1812static int i915_pm_suspend_late(struct device *kdev)
1813{
1814        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1815
1816        /*
1817         * We have a suspend ordering issue with the snd-hda driver also
1818         * requiring our device to be power up. Due to the lack of a
1819         * parent/child relationship we currently solve this with an late
1820         * suspend hook.
1821         *
1822         * FIXME: This should be solved with a special hdmi sink device or
1823         * similar so that power domains can be employed.
1824         */
1825        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1826                return 0;
1827
1828        return i915_drm_suspend_late(dev, false);
1829}
1830
1831static int i915_pm_poweroff_late(struct device *kdev)
1832{
1833        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1834
1835        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1836                return 0;
1837
1838        return i915_drm_suspend_late(dev, true);
1839}
1840
1841static int i915_pm_resume_early(struct device *kdev)
1842{
1843        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1844
1845        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1846                return 0;
1847
1848        return i915_drm_resume_early(dev);
1849}
1850
1851static int i915_pm_resume(struct device *kdev)
1852{
1853        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1854
1855        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1856                return 0;
1857
1858        return i915_drm_resume(dev);
1859}
1860
1861/* freeze: before creating the hibernation_image */
1862static int i915_pm_freeze(struct device *kdev)
1863{
1864        int ret;
1865
1866        ret = i915_pm_suspend(kdev);
1867        if (ret)
1868                return ret;
1869
1870        ret = i915_gem_freeze(kdev_to_i915(kdev));
1871        if (ret)
1872                return ret;
1873
1874        return 0;
1875}
1876
1877static int i915_pm_freeze_late(struct device *kdev)
1878{
1879        int ret;
1880
1881        ret = i915_pm_suspend_late(kdev);
1882        if (ret)
1883                return ret;
1884
1885        ret = i915_gem_freeze_late(kdev_to_i915(kdev));
1886        if (ret)
1887                return ret;
1888
1889        return 0;
1890}
1891
1892/* thaw: called after creating the hibernation image, but before turning off. */
1893static int i915_pm_thaw_early(struct device *kdev)
1894{
1895        return i915_pm_resume_early(kdev);
1896}
1897
1898static int i915_pm_thaw(struct device *kdev)
1899{
1900        return i915_pm_resume(kdev);
1901}
1902
1903/* restore: called after loading the hibernation image. */
1904static int i915_pm_restore_early(struct device *kdev)
1905{
1906        return i915_pm_resume_early(kdev);
1907}
1908
1909static int i915_pm_restore(struct device *kdev)
1910{
1911        return i915_pm_resume(kdev);
1912}
1913
1914/*
1915 * Save all Gunit registers that may be lost after a D3 and a subsequent
1916 * S0i[R123] transition. The list of registers needing a save/restore is
1917 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
1918 * registers in the following way:
1919 * - Driver: saved/restored by the driver
1920 * - Punit : saved/restored by the Punit firmware
1921 * - No, w/o marking: no need to save/restore, since the register is R/O or
1922 *                    used internally by the HW in a way that doesn't depend
1923 *                    keeping the content across a suspend/resume.
1924 * - Debug : used for debugging
1925 *
1926 * We save/restore all registers marked with 'Driver', with the following
1927 * exceptions:
1928 * - Registers out of use, including also registers marked with 'Debug'.
1929 *   These have no effect on the driver's operation, so we don't save/restore
1930 *   them to reduce the overhead.
1931 * - Registers that are fully setup by an initialization function called from
1932 *   the resume path. For example many clock gating and RPS/RC6 registers.
1933 * - Registers that provide the right functionality with their reset defaults.
1934 *
1935 * TODO: Except for registers that based on the above 3 criteria can be safely
1936 * ignored, we save/restore all others, practically treating the HW context as
1937 * a black-box for the driver. Further investigation is needed to reduce the
1938 * saved/restored registers even further, by following the same 3 criteria.
1939 */
1940static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
1941{
1942        struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
1943        int i;
1944
1945        /* GAM 0x4000-0x4770 */
1946        s->wr_watermark         = I915_READ(GEN7_WR_WATERMARK);
1947        s->gfx_prio_ctrl        = I915_READ(GEN7_GFX_PRIO_CTRL);
1948        s->arb_mode             = I915_READ(ARB_MODE);
1949        s->gfx_pend_tlb0        = I915_READ(GEN7_GFX_PEND_TLB0);
1950        s->gfx_pend_tlb1        = I915_READ(GEN7_GFX_PEND_TLB1);
1951
1952        for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
1953                s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
1954
1955        s->media_max_req_count  = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
1956        s->gfx_max_req_count    = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
1957
1958        s->render_hwsp          = I915_READ(RENDER_HWS_PGA_GEN7);
1959        s->ecochk               = I915_READ(GAM_ECOCHK);
1960        s->bsd_hwsp             = I915_READ(BSD_HWS_PGA_GEN7);
1961        s->blt_hwsp             = I915_READ(BLT_HWS_PGA_GEN7);
1962
1963        s->tlb_rd_addr          = I915_READ(GEN7_TLB_RD_ADDR);
1964
1965        /* MBC 0x9024-0x91D0, 0x8500 */
1966        s->g3dctl               = I915_READ(VLV_G3DCTL);
1967        s->gsckgctl             = I915_READ(VLV_GSCKGCTL);
1968        s->mbctl                = I915_READ(GEN6_MBCTL);
1969
1970        /* GCP 0x9400-0x9424, 0x8100-0x810C */
1971        s->ucgctl1              = I915_READ(GEN6_UCGCTL1);
1972        s->ucgctl3              = I915_READ(GEN6_UCGCTL3);
1973        s->rcgctl1              = I915_READ(GEN6_RCGCTL1);
1974        s->rcgctl2              = I915_READ(GEN6_RCGCTL2);
1975        s->rstctl               = I915_READ(GEN6_RSTCTL);
1976        s->misccpctl            = I915_READ(GEN7_MISCCPCTL);
1977
1978        /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
1979        s->gfxpause             = I915_READ(GEN6_GFXPAUSE);
1980        s->rpdeuhwtc            = I915_READ(GEN6_RPDEUHWTC);
1981        s->rpdeuc               = I915_READ(GEN6_RPDEUC);
1982        s->ecobus               = I915_READ(ECOBUS);
1983        s->pwrdwnupctl          = I915_READ(VLV_PWRDWNUPCTL);
1984        s->rp_down_timeout      = I915_READ(GEN6_RP_DOWN_TIMEOUT);
1985        s->rp_deucsw            = I915_READ(GEN6_RPDEUCSW);
1986        s->rcubmabdtmr          = I915_READ(GEN6_RCUBMABDTMR);
1987        s->rcedata              = I915_READ(VLV_RCEDATA);
1988        s->spare2gh             = I915_READ(VLV_SPAREG2H);
1989
1990        /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
1991        s->gt_imr               = I915_READ(GTIMR);
1992        s->gt_ier               = I915_READ(GTIER);
1993        s->pm_imr               = I915_READ(GEN6_PMIMR);
1994        s->pm_ier               = I915_READ(GEN6_PMIER);
1995
1996        for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
1997                s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
1998
1999        /* GT SA CZ domain, 0x100000-0x138124 */
2000        s->tilectl              = I915_READ(TILECTL);
2001        s->gt_fifoctl           = I915_READ(GTFIFOCTL);
2002        s->gtlc_wake_ctrl       = I915_READ(VLV_GTLC_WAKE_CTRL);
2003        s->gtlc_survive         = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2004        s->pmwgicz              = I915_READ(VLV_PMWGICZ);
2005
2006        /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2007        s->gu_ctl0              = I915_READ(VLV_GU_CTL0);
2008        s->gu_ctl1              = I915_READ(VLV_GU_CTL1);
2009        s->pcbr                 = I915_READ(VLV_PCBR);
2010        s->clock_gate_dis2      = I915_READ(VLV_GUNIT_CLOCK_GATE2);
2011
2012        /*
2013         * Not saving any of:
2014         * DFT,         0x9800-0x9EC0
2015         * SARB,        0xB000-0xB1FC
2016         * GAC,         0x5208-0x524C, 0x14000-0x14C000
2017         * PCI CFG
2018         */
2019}
2020
2021static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2022{
2023        struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
2024        u32 val;
2025        int i;
2026
2027        /* GAM 0x4000-0x4770 */
2028        I915_WRITE(GEN7_WR_WATERMARK,   s->wr_watermark);
2029        I915_WRITE(GEN7_GFX_PRIO_CTRL,  s->gfx_prio_ctrl);
2030        I915_WRITE(ARB_MODE,            s->arb_mode | (0xffff << 16));
2031        I915_WRITE(GEN7_GFX_PEND_TLB0,  s->gfx_pend_tlb0);
2032        I915_WRITE(GEN7_GFX_PEND_TLB1,  s->gfx_pend_tlb1);
2033
2034        for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
2035                I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
2036
2037        I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
2038        I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
2039
2040        I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
2041        I915_WRITE(GAM_ECOCHK,          s->ecochk);
2042        I915_WRITE(BSD_HWS_PGA_GEN7,    s->bsd_hwsp);
2043        I915_WRITE(BLT_HWS_PGA_GEN7,    s->blt_hwsp);
2044
2045        I915_WRITE(GEN7_TLB_RD_ADDR,    s->tlb_rd_addr);
2046
2047        /* MBC 0x9024-0x91D0, 0x8500 */
2048        I915_WRITE(VLV_G3DCTL,          s->g3dctl);
2049        I915_WRITE(VLV_GSCKGCTL,        s->gsckgctl);
2050        I915_WRITE(GEN6_MBCTL,          s->mbctl);
2051
2052        /* GCP 0x9400-0x9424, 0x8100-0x810C */
2053        I915_WRITE(GEN6_UCGCTL1,        s->ucgctl1);
2054        I915_WRITE(GEN6_UCGCTL3,        s->ucgctl3);
2055        I915_WRITE(GEN6_RCGCTL1,        s->rcgctl1);
2056        I915_WRITE(GEN6_RCGCTL2,        s->rcgctl2);
2057        I915_WRITE(GEN6_RSTCTL,         s->rstctl);
2058        I915_WRITE(GEN7_MISCCPCTL,      s->misccpctl);
2059
2060        /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2061        I915_WRITE(GEN6_GFXPAUSE,       s->gfxpause);
2062        I915_WRITE(GEN6_RPDEUHWTC,      s->rpdeuhwtc);
2063        I915_WRITE(GEN6_RPDEUC,         s->rpdeuc);
2064        I915_WRITE(ECOBUS,              s->ecobus);
2065        I915_WRITE(VLV_PWRDWNUPCTL,     s->pwrdwnupctl);
2066        I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
2067        I915_WRITE(GEN6_RPDEUCSW,       s->rp_deucsw);
2068        I915_WRITE(GEN6_RCUBMABDTMR,    s->rcubmabdtmr);
2069        I915_WRITE(VLV_RCEDATA,         s->rcedata);
2070        I915_WRITE(VLV_SPAREG2H,        s->spare2gh);
2071
2072        /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2073        I915_WRITE(GTIMR,               s->gt_imr);
2074        I915_WRITE(GTIER,               s->gt_ier);
2075        I915_WRITE(GEN6_PMIMR,          s->pm_imr);
2076        I915_WRITE(GEN6_PMIER,          s->pm_ier);
2077
2078        for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
2079                I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
2080
2081        /* GT SA CZ domain, 0x100000-0x138124 */
2082        I915_WRITE(TILECTL,                     s->tilectl);
2083        I915_WRITE(GTFIFOCTL,                   s->gt_fifoctl);
2084        /*
2085         * Preserve the GT allow wake and GFX force clock bit, they are not
2086         * be restored, as they are used to control the s0ix suspend/resume
2087         * sequence by the caller.
2088         */
2089        val = I915_READ(VLV_GTLC_WAKE_CTRL);
2090        val &= VLV_GTLC_ALLOWWAKEREQ;
2091        val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
2092        I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2093
2094        val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2095        val &= VLV_GFX_CLK_FORCE_ON_BIT;
2096        val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
2097        I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2098
2099        I915_WRITE(VLV_PMWGICZ,                 s->pmwgicz);
2100
2101        /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2102        I915_WRITE(VLV_GU_CTL0,                 s->gu_ctl0);
2103        I915_WRITE(VLV_GU_CTL1,                 s->gu_ctl1);
2104        I915_WRITE(VLV_PCBR,                    s->pcbr);
2105        I915_WRITE(VLV_GUNIT_CLOCK_GATE2,       s->clock_gate_dis2);
2106}
2107
2108int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
2109{
2110        u32 val;
2111        int err;
2112
2113        val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2114        val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
2115        if (force_on)
2116                val |= VLV_GFX_CLK_FORCE_ON_BIT;
2117        I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2118
2119        if (!force_on)
2120                return 0;
2121
2122        err = intel_wait_for_register(dev_priv,
2123                                      VLV_GTLC_SURVIVABILITY_REG,
2124                                      VLV_GFX_CLK_STATUS_BIT,
2125                                      VLV_GFX_CLK_STATUS_BIT,
2126                                      20);
2127        if (err)
2128                DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
2129                          I915_READ(VLV_GTLC_SURVIVABILITY_REG));
2130
2131        return err;
2132}
2133
2134static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
2135{
2136        u32 val;
2137        int err = 0;
2138
2139        val = I915_READ(VLV_GTLC_WAKE_CTRL);
2140        val &= ~VLV_GTLC_ALLOWWAKEREQ;
2141        if (allow)
2142                val |= VLV_GTLC_ALLOWWAKEREQ;
2143        I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2144        POSTING_READ(VLV_GTLC_WAKE_CTRL);
2145
2146        err = intel_wait_for_register(dev_priv,
2147                                      VLV_GTLC_PW_STATUS,
2148                                      VLV_GTLC_ALLOWWAKEACK,
2149                                      allow,
2150                                      1);
2151        if (err)
2152                DRM_ERROR("timeout disabling GT waking\n");
2153
2154        return err;
2155}
2156
2157static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
2158                                 bool wait_for_on)
2159{
2160        u32 mask;
2161        u32 val;
2162        int err;
2163
2164        mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
2165        val = wait_for_on ? mask : 0;
2166        if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
2167                return 0;
2168
2169        DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
2170                      onoff(wait_for_on),
2171                      I915_READ(VLV_GTLC_PW_STATUS));
2172
2173        /*
2174         * RC6 transitioning can be delayed up to 2 msec (see
2175         * valleyview_enable_rps), use 3 msec for safety.
2176         */
2177        err = intel_wait_for_register(dev_priv,
2178                                      VLV_GTLC_PW_STATUS, mask, val,
2179                                      3);
2180        if (err)
2181                DRM_ERROR("timeout waiting for GT wells to go %s\n",
2182                          onoff(wait_for_on));
2183
2184        return err;
2185}
2186
2187static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
2188{
2189        if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
2190                return;
2191
2192        DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
2193        I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
2194}
2195
2196static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
2197{
2198        u32 mask;
2199        int err;
2200
2201        /*
2202         * Bspec defines the following GT well on flags as debug only, so
2203         * don't treat them as hard failures.
2204         */
2205        (void)vlv_wait_for_gt_wells(dev_priv, false);
2206
2207        mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
2208        WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
2209
2210        vlv_check_no_gt_access(dev_priv);
2211
2212        err = vlv_force_gfx_clock(dev_priv, true);
2213        if (err)
2214                goto err1;
2215
2216        err = vlv_allow_gt_wake(dev_priv, false);
2217        if (err)
2218                goto err2;
2219
2220        if (!IS_CHERRYVIEW(dev_priv))
2221                vlv_save_gunit_s0ix_state(dev_priv);
2222
2223        err = vlv_force_gfx_clock(dev_priv, false);
2224        if (err)
2225                goto err2;
2226
2227        return 0;
2228
2229err2:
2230        /* For safety always re-enable waking and disable gfx clock forcing */
2231        vlv_allow_gt_wake(dev_priv, true);
2232err1:
2233        vlv_force_gfx_clock(dev_priv, false);
2234
2235        return err;
2236}
2237
2238static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
2239                                bool rpm_resume)
2240{
2241        struct drm_device *dev = &dev_priv->drm;
2242        int err;
2243        int ret;
2244
2245        /*
2246         * If any of the steps fail just try to continue, that's the best we
2247         * can do at this point. Return the first error code (which will also
2248         * leave RPM permanently disabled).
2249         */
2250        ret = vlv_force_gfx_clock(dev_priv, true);
2251
2252        if (!IS_CHERRYVIEW(dev_priv))
2253                vlv_restore_gunit_s0ix_state(dev_priv);
2254
2255        err = vlv_allow_gt_wake(dev_priv, true);
2256        if (!ret)
2257                ret = err;
2258
2259        err = vlv_force_gfx_clock(dev_priv, false);
2260        if (!ret)
2261                ret = err;
2262
2263        vlv_check_no_gt_access(dev_priv);
2264
2265        if (rpm_resume) {
2266                intel_init_clock_gating(dev);
2267                i915_gem_restore_fences(dev);
2268        }
2269
2270        return ret;
2271}
2272
2273static int intel_runtime_suspend(struct device *kdev)
2274{
2275        struct pci_dev *pdev = to_pci_dev(kdev);
2276        struct drm_device *dev = pci_get_drvdata(pdev);
2277        struct drm_i915_private *dev_priv = to_i915(dev);
2278        int ret;
2279
2280        if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
2281                return -ENODEV;
2282
2283        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2284                return -ENODEV;
2285
2286        DRM_DEBUG_KMS("Suspending device\n");
2287
2288        /*
2289         * We could deadlock here in case another thread holding struct_mutex
2290         * calls RPM suspend concurrently, since the RPM suspend will wait
2291         * first for this RPM suspend to finish. In this case the concurrent
2292         * RPM resume will be followed by its RPM suspend counterpart. Still
2293         * for consistency return -EAGAIN, which will reschedule this suspend.
2294         */
2295        if (!mutex_trylock(&dev->struct_mutex)) {
2296                DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
2297                /*
2298                 * Bump the expiration timestamp, otherwise the suspend won't
2299                 * be rescheduled.
2300                 */
2301                pm_runtime_mark_last_busy(kdev);
2302
2303                return -EAGAIN;
2304        }
2305
2306        disable_rpm_wakeref_asserts(dev_priv);
2307
2308        /*
2309         * We are safe here against re-faults, since the fault handler takes
2310         * an RPM reference.
2311         */
2312        i915_gem_release_all_mmaps(dev_priv);
2313        mutex_unlock(&dev->struct_mutex);
2314
2315        intel_guc_suspend(dev);
2316
2317        intel_runtime_pm_disable_interrupts(dev_priv);
2318
2319        ret = 0;
2320        if (IS_BROXTON(dev_priv)) {
2321                bxt_display_core_uninit(dev_priv);
2322                bxt_enable_dc9(dev_priv);
2323        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2324                hsw_enable_pc8(dev_priv);
2325        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2326                ret = vlv_suspend_complete(dev_priv);
2327        }
2328
2329        if (ret) {
2330                DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
2331                intel_runtime_pm_enable_interrupts(dev_priv);
2332
2333                enable_rpm_wakeref_asserts(dev_priv);
2334
2335                return ret;
2336        }
2337
2338        intel_uncore_forcewake_reset(dev_priv, false);
2339
2340        enable_rpm_wakeref_asserts(dev_priv);
2341        WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2342
2343        if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv))
2344                DRM_ERROR("Unclaimed access detected prior to suspending\n");
2345
2346        dev_priv->pm.suspended = true;
2347
2348        /*
2349         * FIXME: We really should find a document that references the arguments
2350         * used below!
2351         */
2352        if (IS_BROADWELL(dev_priv)) {
2353                /*
2354                 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
2355                 * being detected, and the call we do at intel_runtime_resume()
2356                 * won't be able to restore them. Since PCI_D3hot matches the
2357                 * actual specification and appears to be working, use it.
2358                 */
2359                intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
2360        } else {
2361                /*
2362                 * current versions of firmware which depend on this opregion
2363                 * notification have repurposed the D1 definition to mean
2364                 * "runtime suspended" vs. what you would normally expect (D3)
2365                 * to distinguish it from notifications that might be sent via
2366                 * the suspend path.
2367                 */
2368                intel_opregion_notify_adapter(dev_priv, PCI_D1);
2369        }
2370
2371        assert_forcewakes_inactive(dev_priv);
2372
2373        if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
2374                intel_hpd_poll_init(dev_priv);
2375
2376        DRM_DEBUG_KMS("Device suspended\n");
2377        return 0;
2378}
2379
2380static int intel_runtime_resume(struct device *kdev)
2381{
2382        struct pci_dev *pdev = to_pci_dev(kdev);
2383        struct drm_device *dev = pci_get_drvdata(pdev);
2384        struct drm_i915_private *dev_priv = to_i915(dev);
2385        int ret = 0;
2386
2387        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2388                return -ENODEV;
2389
2390        DRM_DEBUG_KMS("Resuming device\n");
2391
2392        WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2393        disable_rpm_wakeref_asserts(dev_priv);
2394
2395        intel_opregion_notify_adapter(dev_priv, PCI_D0);
2396        dev_priv->pm.suspended = false;
2397        if (intel_uncore_unclaimed_mmio(dev_priv))
2398                DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
2399
2400        intel_guc_resume(dev);
2401
2402        if (IS_GEN6(dev_priv))
2403                intel_init_pch_refclk(dev);
2404
2405        if (IS_BROXTON(dev)) {
2406                bxt_disable_dc9(dev_priv);
2407                bxt_display_core_init(dev_priv, true);
2408                if (dev_priv->csr.dmc_payload &&
2409                    (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2410                        gen9_enable_dc5(dev_priv);
2411        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2412                hsw_disable_pc8(dev_priv);
2413        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2414                ret = vlv_resume_prepare(dev_priv, true);
2415        }
2416
2417        /*
2418         * No point of rolling back things in case of an error, as the best
2419         * we can do is to hope that things will still work (and disable RPM).
2420         */
2421        i915_gem_init_swizzling(dev);
2422
2423        intel_runtime_pm_enable_interrupts(dev_priv);
2424
2425        /*
2426         * On VLV/CHV display interrupts are part of the display
2427         * power well, so hpd is reinitialized from there. For
2428         * everyone else do it here.
2429         */
2430        if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
2431                intel_hpd_init(dev_priv);
2432
2433        enable_rpm_wakeref_asserts(dev_priv);
2434
2435        if (ret)
2436                DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
2437        else
2438                DRM_DEBUG_KMS("Device resumed\n");
2439
2440        return ret;
2441}
2442
2443const struct dev_pm_ops i915_pm_ops = {
2444        /*
2445         * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
2446         * PMSG_RESUME]
2447         */
2448        .suspend = i915_pm_suspend,
2449        .suspend_late = i915_pm_suspend_late,
2450        .resume_early = i915_pm_resume_early,
2451        .resume = i915_pm_resume,
2452
2453        /*
2454         * S4 event handlers
2455         * @freeze, @freeze_late    : called (1) before creating the
2456         *                            hibernation image [PMSG_FREEZE] and
2457         *                            (2) after rebooting, before restoring
2458         *                            the image [PMSG_QUIESCE]
2459         * @thaw, @thaw_early       : called (1) after creating the hibernation
2460         *                            image, before writing it [PMSG_THAW]
2461         *                            and (2) after failing to create or
2462         *                            restore the image [PMSG_RECOVER]
2463         * @poweroff, @poweroff_late: called after writing the hibernation
2464         *                            image, before rebooting [PMSG_HIBERNATE]
2465         * @restore, @restore_early : called after rebooting and restoring the
2466         *                            hibernation image [PMSG_RESTORE]
2467         */
2468        .freeze = i915_pm_freeze,
2469        .freeze_late = i915_pm_freeze_late,
2470        .thaw_early = i915_pm_thaw_early,
2471        .thaw = i915_pm_thaw,
2472        .poweroff = i915_pm_suspend,
2473        .poweroff_late = i915_pm_poweroff_late,
2474        .restore_early = i915_pm_restore_early,
2475        .restore = i915_pm_restore,
2476
2477        /* S0ix (via runtime suspend) event handlers */
2478        .runtime_suspend = intel_runtime_suspend,
2479        .runtime_resume = intel_runtime_resume,
2480};
2481
2482static const struct vm_operations_struct i915_gem_vm_ops = {
2483        .fault = i915_gem_fault,
2484        .open = drm_gem_vm_open,
2485        .close = drm_gem_vm_close,
2486};
2487
2488static const struct file_operations i915_driver_fops = {
2489        .owner = THIS_MODULE,
2490        .open = drm_open,
2491        .release = drm_release,
2492        .unlocked_ioctl = drm_ioctl,
2493        .mmap = drm_gem_mmap,
2494        .poll = drm_poll,
2495        .read = drm_read,
2496#ifdef CONFIG_COMPAT
2497        .compat_ioctl = i915_compat_ioctl,
2498#endif
2499        .llseek = noop_llseek,
2500};
2501
2502static int
2503i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
2504                          struct drm_file *file)
2505{
2506        return -ENODEV;
2507}
2508
2509static const struct drm_ioctl_desc i915_ioctls[] = {
2510        DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2511        DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
2512        DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
2513        DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
2514        DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
2515        DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
2516        DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
2517        DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2518        DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2519        DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2520        DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2521        DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
2522        DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2523        DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2524        DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  drm_noop, DRM_AUTH),
2525        DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
2526        DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2527        DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2528        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
2529        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
2530        DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2531        DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2532        DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2533        DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
2534        DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
2535        DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2536        DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2537        DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2538        DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
2539        DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
2540        DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
2541        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
2542        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
2543        DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
2544        DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
2545        DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
2546        DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
2547        DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
2548        DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
2549        DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
2550        DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2551        DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2552        DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
2553        DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
2554        DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2555        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
2556        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
2557        DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
2558        DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
2559        DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
2560        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
2561        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
2562};
2563
2564static struct drm_driver driver = {
2565        /* Don't use MTRRs here; the Xserver or userspace app should
2566         * deal with them for Intel hardware.
2567         */
2568        .driver_features =
2569            DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
2570            DRIVER_RENDER | DRIVER_MODESET,
2571        .open = i915_driver_open,
2572        .lastclose = i915_driver_lastclose,
2573        .preclose = i915_driver_preclose,
2574        .postclose = i915_driver_postclose,
2575        .set_busid = drm_pci_set_busid,
2576
2577        .gem_close_object = i915_gem_close_object,
2578        .gem_free_object = i915_gem_free_object,
2579        .gem_vm_ops = &i915_gem_vm_ops,
2580
2581        .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
2582        .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
2583        .gem_prime_export = i915_gem_prime_export,
2584        .gem_prime_import = i915_gem_prime_import,
2585
2586        .dumb_create = i915_gem_dumb_create,
2587        .dumb_map_offset = i915_gem_mmap_gtt,
2588        .dumb_destroy = drm_gem_dumb_destroy,
2589        .ioctls = i915_ioctls,
2590        .num_ioctls = ARRAY_SIZE(i915_ioctls),
2591        .fops = &i915_driver_fops,
2592        .name = DRIVER_NAME,
2593        .desc = DRIVER_DESC,
2594        .date = DRIVER_DATE,
2595        .major = DRIVER_MAJOR,
2596        .minor = DRIVER_MINOR,
2597        .patchlevel = DRIVER_PATCHLEVEL,
2598};
Note: See TracBrowser for help on using the repository browser.