source: src/linux/universal/linux-4.9/drivers/gpu/drm/i915/i915_drv.c @ 31574

Last change on this file since 31574 was 31574, checked in by brainslayer, 5 months ago

kernel 4.9 update

File size: 73.3 KB
Line 
1/* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
2 */
3/*
4 *
5 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 */
29
30#include <linux/acpi.h>
31#include <linux/device.h>
32#include <linux/oom.h>
33#include <linux/module.h>
34#include <linux/pci.h>
35#include <linux/pm.h>
36#include <linux/pm_runtime.h>
37#include <linux/pnp.h>
38#include <linux/slab.h>
39#include <linux/vgaarb.h>
40#include <linux/vga_switcheroo.h>
41#include <linux/vt.h>
42#include <acpi/video.h>
43
44#include <drm/drmP.h>
45#include <drm/drm_crtc_helper.h>
46#include <drm/i915_drm.h>
47
48#include "i915_drv.h"
49#include "i915_trace.h"
50#include "i915_vgpu.h"
51#include "intel_drv.h"
52
53static struct drm_driver driver;
54
55static unsigned int i915_load_fail_count;
56
57bool __i915_inject_load_failure(const char *func, int line)
58{
59        if (i915_load_fail_count >= i915.inject_load_failure)
60                return false;
61
62        if (++i915_load_fail_count == i915.inject_load_failure) {
63                DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
64                         i915.inject_load_failure, func, line);
65                return true;
66        }
67
68        return false;
69}
70
71#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
72#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
73                    "providing the dmesg log by booting with drm.debug=0xf"
74
75void
76__i915_printk(struct drm_i915_private *dev_priv, const char *level,
77              const char *fmt, ...)
78{
79        static bool shown_bug_once;
80        struct device *kdev = dev_priv->drm.dev;
81        bool is_error = level[1] <= KERN_ERR[1];
82        bool is_debug = level[1] == KERN_DEBUG[1];
83        struct va_format vaf;
84        va_list args;
85
86        if (is_debug && !(drm_debug & DRM_UT_DRIVER))
87                return;
88
89        va_start(args, fmt);
90
91        vaf.fmt = fmt;
92        vaf.va = &args;
93
94        dev_printk(level, kdev, "[" DRM_NAME ":%ps] %pV",
95                   __builtin_return_address(0), &vaf);
96
97        if (is_error && !shown_bug_once) {
98                dev_notice(kdev, "%s", FDO_BUG_MSG);
99                shown_bug_once = true;
100        }
101
102        va_end(args);
103}
104
105static bool i915_error_injected(struct drm_i915_private *dev_priv)
106{
107        return i915.inject_load_failure &&
108               i915_load_fail_count == i915.inject_load_failure;
109}
110
111#define i915_load_error(dev_priv, fmt, ...)                                  \
112        __i915_printk(dev_priv,                                              \
113                      i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
114                      fmt, ##__VA_ARGS__)
115
116
117static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
118{
119        enum intel_pch ret = PCH_NOP;
120
121        /*
122         * In a virtualized passthrough environment we can be in a
123         * setup where the ISA bridge is not able to be passed through.
124         * In this case, a south bridge can be emulated and we have to
125         * make an educated guess as to which PCH is really there.
126         */
127
128        if (IS_GEN5(dev)) {
129                ret = PCH_IBX;
130                DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
131        } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
132                ret = PCH_CPT;
133                DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
134        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
135                ret = PCH_LPT;
136                DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
137        } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
138                ret = PCH_SPT;
139                DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
140        }
141
142        return ret;
143}
144
145static void intel_detect_pch(struct drm_device *dev)
146{
147        struct drm_i915_private *dev_priv = to_i915(dev);
148        struct pci_dev *pch = NULL;
149
150        /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
151         * (which really amounts to a PCH but no South Display).
152         */
153        if (INTEL_INFO(dev)->num_pipes == 0) {
154                dev_priv->pch_type = PCH_NOP;
155                return;
156        }
157
158        /*
159         * The reason to probe ISA bridge instead of Dev31:Fun0 is to
160         * make graphics device passthrough work easy for VMM, that only
161         * need to expose ISA bridge to let driver know the real hardware
162         * underneath. This is a requirement from virtualization team.
163         *
164         * In some virtualized environments (e.g. XEN), there is irrelevant
165         * ISA bridge in the system. To work reliably, we should scan trhough
166         * all the ISA bridge devices and check for the first match, instead
167         * of only checking the first one.
168         */
169        while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
170                if (pch->vendor == PCI_VENDOR_ID_INTEL) {
171                        unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
172                        dev_priv->pch_id = id;
173
174                        if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
175                                dev_priv->pch_type = PCH_IBX;
176                                DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
177                                WARN_ON(!IS_GEN5(dev));
178                        } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
179                                dev_priv->pch_type = PCH_CPT;
180                                DRM_DEBUG_KMS("Found CougarPoint PCH\n");
181                                WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
182                        } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
183                                /* PantherPoint is CPT compatible */
184                                dev_priv->pch_type = PCH_CPT;
185                                DRM_DEBUG_KMS("Found PantherPoint PCH\n");
186                                WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
187                        } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
188                                dev_priv->pch_type = PCH_LPT;
189                                DRM_DEBUG_KMS("Found LynxPoint PCH\n");
190                                WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
191                                WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
192                        } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
193                                dev_priv->pch_type = PCH_LPT;
194                                DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
195                                WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
196                                WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
197                        } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
198                                dev_priv->pch_type = PCH_SPT;
199                                DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
200                                WARN_ON(!IS_SKYLAKE(dev) &&
201                                        !IS_KABYLAKE(dev));
202                        } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
203                                dev_priv->pch_type = PCH_SPT;
204                                DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
205                                WARN_ON(!IS_SKYLAKE(dev) &&
206                                        !IS_KABYLAKE(dev));
207                        } else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
208                                dev_priv->pch_type = PCH_KBP;
209                                DRM_DEBUG_KMS("Found KabyPoint PCH\n");
210                                WARN_ON(!IS_KABYLAKE(dev));
211                        } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
212                                   (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
213                                   ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
214                                    pch->subsystem_vendor ==
215                                            PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
216                                    pch->subsystem_device ==
217                                            PCI_SUBDEVICE_ID_QEMU)) {
218                                dev_priv->pch_type = intel_virt_detect_pch(dev);
219                        } else
220                                continue;
221
222                        break;
223                }
224        }
225        if (!pch)
226                DRM_DEBUG_KMS("No PCH found.\n");
227
228        pci_dev_put(pch);
229}
230
231static int i915_getparam(struct drm_device *dev, void *data,
232                         struct drm_file *file_priv)
233{
234        struct drm_i915_private *dev_priv = to_i915(dev);
235        struct pci_dev *pdev = dev_priv->drm.pdev;
236        drm_i915_getparam_t *param = data;
237        int value;
238
239        switch (param->param) {
240        case I915_PARAM_IRQ_ACTIVE:
241        case I915_PARAM_ALLOW_BATCHBUFFER:
242        case I915_PARAM_LAST_DISPATCH:
243                /* Reject all old ums/dri params. */
244                return -ENODEV;
245        case I915_PARAM_CHIPSET_ID:
246                value = pdev->device;
247                break;
248        case I915_PARAM_REVISION:
249                value = pdev->revision;
250                break;
251        case I915_PARAM_NUM_FENCES_AVAIL:
252                value = dev_priv->num_fence_regs;
253                break;
254        case I915_PARAM_HAS_OVERLAY:
255                value = dev_priv->overlay ? 1 : 0;
256                break;
257        case I915_PARAM_HAS_BSD:
258                value = intel_engine_initialized(&dev_priv->engine[VCS]);
259                break;
260        case I915_PARAM_HAS_BLT:
261                value = intel_engine_initialized(&dev_priv->engine[BCS]);
262                break;
263        case I915_PARAM_HAS_VEBOX:
264                value = intel_engine_initialized(&dev_priv->engine[VECS]);
265                break;
266        case I915_PARAM_HAS_BSD2:
267                value = intel_engine_initialized(&dev_priv->engine[VCS2]);
268                break;
269        case I915_PARAM_HAS_EXEC_CONSTANTS:
270                value = INTEL_GEN(dev_priv) >= 4;
271                break;
272        case I915_PARAM_HAS_LLC:
273                value = HAS_LLC(dev_priv);
274                break;
275        case I915_PARAM_HAS_WT:
276                value = HAS_WT(dev_priv);
277                break;
278        case I915_PARAM_HAS_ALIASING_PPGTT:
279                value = USES_PPGTT(dev_priv);
280                break;
281        case I915_PARAM_HAS_SEMAPHORES:
282                value = i915.semaphores;
283                break;
284        case I915_PARAM_HAS_SECURE_BATCHES:
285                value = capable(CAP_SYS_ADMIN);
286                break;
287        case I915_PARAM_CMD_PARSER_VERSION:
288                value = i915_cmd_parser_get_version(dev_priv);
289                break;
290        case I915_PARAM_SUBSLICE_TOTAL:
291                value = sseu_subslice_total(&INTEL_INFO(dev_priv)->sseu);
292                if (!value)
293                        return -ENODEV;
294                break;
295        case I915_PARAM_EU_TOTAL:
296                value = INTEL_INFO(dev_priv)->sseu.eu_total;
297                if (!value)
298                        return -ENODEV;
299                break;
300        case I915_PARAM_HAS_GPU_RESET:
301                value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
302                break;
303        case I915_PARAM_HAS_RESOURCE_STREAMER:
304                value = HAS_RESOURCE_STREAMER(dev_priv);
305                break;
306        case I915_PARAM_HAS_POOLED_EU:
307                value = HAS_POOLED_EU(dev_priv);
308                break;
309        case I915_PARAM_MIN_EU_IN_POOL:
310                value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool;
311                break;
312        case I915_PARAM_MMAP_GTT_VERSION:
313                /* Though we've started our numbering from 1, and so class all
314                 * earlier versions as 0, in effect their value is undefined as
315                 * the ioctl will report EINVAL for the unknown param!
316                 */
317                value = i915_gem_mmap_gtt_version();
318                break;
319        case I915_PARAM_MMAP_VERSION:
320                /* Remember to bump this if the version changes! */
321        case I915_PARAM_HAS_GEM:
322        case I915_PARAM_HAS_PAGEFLIPPING:
323        case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */
324        case I915_PARAM_HAS_RELAXED_FENCING:
325        case I915_PARAM_HAS_COHERENT_RINGS:
326        case I915_PARAM_HAS_RELAXED_DELTA:
327        case I915_PARAM_HAS_GEN7_SOL_RESET:
328        case I915_PARAM_HAS_WAIT_TIMEOUT:
329        case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
330        case I915_PARAM_HAS_PINNED_BATCHES:
331        case I915_PARAM_HAS_EXEC_NO_RELOC:
332        case I915_PARAM_HAS_EXEC_HANDLE_LUT:
333        case I915_PARAM_HAS_COHERENT_PHYS_GTT:
334        case I915_PARAM_HAS_EXEC_SOFTPIN:
335                /* For the time being all of these are always true;
336                 * if some supported hardware does not have one of these
337                 * features this value needs to be provided from
338                 * INTEL_INFO(), a feature macro, or similar.
339                 */
340                value = 1;
341                break;
342        default:
343                DRM_DEBUG("Unknown parameter %d\n", param->param);
344                return -EINVAL;
345        }
346
347        if (put_user(value, param->value))
348                return -EFAULT;
349
350        return 0;
351}
352
353static int i915_get_bridge_dev(struct drm_device *dev)
354{
355        struct drm_i915_private *dev_priv = to_i915(dev);
356
357        dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
358        if (!dev_priv->bridge_dev) {
359                DRM_ERROR("bridge device not found\n");
360                return -1;
361        }
362        return 0;
363}
364
365/* Allocate space for the MCH regs if needed, return nonzero on error */
366static int
367intel_alloc_mchbar_resource(struct drm_device *dev)
368{
369        struct drm_i915_private *dev_priv = to_i915(dev);
370        int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
371        u32 temp_lo, temp_hi = 0;
372        u64 mchbar_addr;
373        int ret;
374
375        if (INTEL_INFO(dev)->gen >= 4)
376                pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
377        pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
378        mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
379
380        /* If ACPI doesn't have it, assume we need to allocate it ourselves */
381#ifdef CONFIG_PNP
382        if (mchbar_addr &&
383            pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
384                return 0;
385#endif
386
387        /* Get some space for it */
388        dev_priv->mch_res.name = "i915 MCHBAR";
389        dev_priv->mch_res.flags = IORESOURCE_MEM;
390        ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
391                                     &dev_priv->mch_res,
392                                     MCHBAR_SIZE, MCHBAR_SIZE,
393                                     PCIBIOS_MIN_MEM,
394                                     0, pcibios_align_resource,
395                                     dev_priv->bridge_dev);
396        if (ret) {
397                DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
398                dev_priv->mch_res.start = 0;
399                return ret;
400        }
401
402        if (INTEL_INFO(dev)->gen >= 4)
403                pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
404                                       upper_32_bits(dev_priv->mch_res.start));
405
406        pci_write_config_dword(dev_priv->bridge_dev, reg,
407                               lower_32_bits(dev_priv->mch_res.start));
408        return 0;
409}
410
411/* Setup MCHBAR if possible, return true if we should disable it again */
412static void
413intel_setup_mchbar(struct drm_device *dev)
414{
415        struct drm_i915_private *dev_priv = to_i915(dev);
416        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
417        u32 temp;
418        bool enabled;
419
420        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
421                return;
422
423        dev_priv->mchbar_need_disable = false;
424
425        if (IS_I915G(dev) || IS_I915GM(dev)) {
426                pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
427                enabled = !!(temp & DEVEN_MCHBAR_EN);
428        } else {
429                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
430                enabled = temp & 1;
431        }
432
433        /* If it's already enabled, don't have to do anything */
434        if (enabled)
435                return;
436
437        if (intel_alloc_mchbar_resource(dev))
438                return;
439
440        dev_priv->mchbar_need_disable = true;
441
442        /* Space is allocated or reserved, so enable it. */
443        if (IS_I915G(dev) || IS_I915GM(dev)) {
444                pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
445                                       temp | DEVEN_MCHBAR_EN);
446        } else {
447                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
448                pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
449        }
450}
451
452static void
453intel_teardown_mchbar(struct drm_device *dev)
454{
455        struct drm_i915_private *dev_priv = to_i915(dev);
456        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
457
458        if (dev_priv->mchbar_need_disable) {
459                if (IS_I915G(dev) || IS_I915GM(dev)) {
460                        u32 deven_val;
461
462                        pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
463                                              &deven_val);
464                        deven_val &= ~DEVEN_MCHBAR_EN;
465                        pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
466                                               deven_val);
467                } else {
468                        u32 mchbar_val;
469
470                        pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
471                                              &mchbar_val);
472                        mchbar_val &= ~1;
473                        pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
474                                               mchbar_val);
475                }
476        }
477
478        if (dev_priv->mch_res.start)
479                release_resource(&dev_priv->mch_res);
480}
481
482/* true = enable decode, false = disable decoder */
483static unsigned int i915_vga_set_decode(void *cookie, bool state)
484{
485        struct drm_device *dev = cookie;
486
487        intel_modeset_vga_set_state(dev, state);
488        if (state)
489                return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
490                       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
491        else
492                return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
493}
494
495static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
496{
497        struct drm_device *dev = pci_get_drvdata(pdev);
498        pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
499
500        if (state == VGA_SWITCHEROO_ON) {
501                pr_info("switched on\n");
502                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
503                /* i915 resume handler doesn't set to D0 */
504                pci_set_power_state(pdev, PCI_D0);
505                i915_resume_switcheroo(dev);
506                dev->switch_power_state = DRM_SWITCH_POWER_ON;
507        } else {
508                pr_info("switched off\n");
509                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
510                i915_suspend_switcheroo(dev, pmm);
511                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
512        }
513}
514
515static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
516{
517        struct drm_device *dev = pci_get_drvdata(pdev);
518
519        /*
520         * FIXME: open_count is protected by drm_global_mutex but that would lead to
521         * locking inversion with the driver load path. And the access here is
522         * completely racy anyway. So don't bother with locking for now.
523         */
524        return dev->open_count == 0;
525}
526
527static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
528        .set_gpu_state = i915_switcheroo_set_state,
529        .reprobe = NULL,
530        .can_switch = i915_switcheroo_can_switch,
531};
532
533static void i915_gem_fini(struct drm_device *dev)
534{
535        struct drm_i915_private *dev_priv = to_i915(dev);
536
537        /*
538         * Neither the BIOS, ourselves or any other kernel
539         * expects the system to be in execlists mode on startup,
540         * so we need to reset the GPU back to legacy mode. And the only
541         * known way to disable logical contexts is through a GPU reset.
542         *
543         * So in order to leave the system in a known default configuration,
544         * always reset the GPU upon unload. Afterwards we then clean up the
545         * GEM state tracking, flushing off the requests and leaving the
546         * system in a known idle state.
547         *
548         * Note that is of the upmost importance that the GPU is idle and
549         * all stray writes are flushed *before* we dismantle the backing
550         * storage for the pinned objects.
551         *
552         * However, since we are uncertain that reseting the GPU on older
553         * machines is a good idea, we don't - just in case it leaves the
554         * machine in an unusable condition.
555         */
556        if (HAS_HW_CONTEXTS(dev)) {
557                int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
558                WARN_ON(reset && reset != -ENODEV);
559        }
560
561        mutex_lock(&dev->struct_mutex);
562        i915_gem_cleanup_engines(dev);
563        i915_gem_context_fini(dev);
564        mutex_unlock(&dev->struct_mutex);
565
566        WARN_ON(!list_empty(&to_i915(dev)->context_list));
567}
568
569static int i915_load_modeset_init(struct drm_device *dev)
570{
571        struct drm_i915_private *dev_priv = to_i915(dev);
572        struct pci_dev *pdev = dev_priv->drm.pdev;
573        int ret;
574
575        if (i915_inject_load_failure())
576                return -ENODEV;
577
578        ret = intel_bios_init(dev_priv);
579        if (ret)
580                DRM_INFO("failed to find VBIOS tables\n");
581
582        /* If we have > 1 VGA cards, then we need to arbitrate access
583         * to the common VGA resources.
584         *
585         * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
586         * then we do not take part in VGA arbitration and the
587         * vga_client_register() fails with -ENODEV.
588         */
589        ret = vga_client_register(pdev, dev, NULL, i915_vga_set_decode);
590        if (ret && ret != -ENODEV)
591                goto out;
592
593        intel_register_dsm_handler();
594
595        ret = vga_switcheroo_register_client(pdev, &i915_switcheroo_ops, false);
596        if (ret)
597                goto cleanup_vga_client;
598
599        /* must happen before intel_power_domains_init_hw() on VLV/CHV */
600        intel_update_rawclk(dev_priv);
601
602        intel_power_domains_init_hw(dev_priv, false);
603
604        intel_csr_ucode_init(dev_priv);
605
606        ret = intel_irq_install(dev_priv);
607        if (ret)
608                goto cleanup_csr;
609
610        intel_setup_gmbus(dev);
611
612        /* Important: The output setup functions called by modeset_init need
613         * working irqs for e.g. gmbus and dp aux transfers. */
614        intel_modeset_init(dev);
615
616        intel_guc_init(dev);
617
618        ret = i915_gem_init(dev);
619        if (ret)
620                goto cleanup_irq;
621
622        intel_modeset_gem_init(dev);
623
624        if (INTEL_INFO(dev)->num_pipes == 0)
625                return 0;
626
627        ret = intel_fbdev_init(dev);
628        if (ret)
629                goto cleanup_gem;
630
631        /* Only enable hotplug handling once the fbdev is fully set up. */
632        intel_hpd_init(dev_priv);
633
634        drm_kms_helper_poll_init(dev);
635
636        return 0;
637
638cleanup_gem:
639        i915_gem_fini(dev);
640cleanup_irq:
641        intel_guc_fini(dev);
642        drm_irq_uninstall(dev);
643        intel_teardown_gmbus(dev);
644cleanup_csr:
645        intel_csr_ucode_fini(dev_priv);
646        intel_power_domains_fini(dev_priv);
647        vga_switcheroo_unregister_client(pdev);
648cleanup_vga_client:
649        vga_client_register(pdev, NULL, NULL, NULL);
650out:
651        return ret;
652}
653
654#if IS_ENABLED(CONFIG_FB)
655static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
656{
657        struct apertures_struct *ap;
658        struct pci_dev *pdev = dev_priv->drm.pdev;
659        struct i915_ggtt *ggtt = &dev_priv->ggtt;
660        bool primary;
661        int ret;
662
663        ap = alloc_apertures(1);
664        if (!ap)
665                return -ENOMEM;
666
667        ap->ranges[0].base = ggtt->mappable_base;
668        ap->ranges[0].size = ggtt->mappable_end;
669
670        primary =
671                pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
672
673        ret = drm_fb_helper_remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
674
675        kfree(ap);
676
677        return ret;
678}
679#else
680static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
681{
682        return 0;
683}
684#endif
685
686#if !defined(CONFIG_VGA_CONSOLE)
687static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
688{
689        return 0;
690}
691#elif !defined(CONFIG_DUMMY_CONSOLE)
692static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
693{
694        return -ENODEV;
695}
696#else
697static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
698{
699        int ret = 0;
700
701        DRM_INFO("Replacing VGA console driver\n");
702
703        console_lock();
704        if (con_is_bound(&vga_con))
705                ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
706        if (ret == 0) {
707                ret = do_unregister_con_driver(&vga_con);
708
709                /* Ignore "already unregistered". */
710                if (ret == -ENODEV)
711                        ret = 0;
712        }
713        console_unlock();
714
715        return ret;
716}
717#endif
718
719static void intel_init_dpio(struct drm_i915_private *dev_priv)
720{
721        /*
722         * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
723         * CHV x1 PHY (DP/HDMI D)
724         * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
725         */
726        if (IS_CHERRYVIEW(dev_priv)) {
727                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
728                DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
729        } else if (IS_VALLEYVIEW(dev_priv)) {
730                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
731        }
732}
733
734static int i915_workqueues_init(struct drm_i915_private *dev_priv)
735{
736        /*
737         * The i915 workqueue is primarily used for batched retirement of
738         * requests (and thus managing bo) once the task has been completed
739         * by the GPU. i915_gem_retire_requests() is called directly when we
740         * need high-priority retirement, such as waiting for an explicit
741         * bo.
742         *
743         * It is also used for periodic low-priority events, such as
744         * idle-timers and recording error state.
745         *
746         * All tasks on the workqueue are expected to acquire the dev mutex
747         * so there is no point in running more than one instance of the
748         * workqueue at any time.  Use an ordered one.
749         */
750        dev_priv->wq = alloc_ordered_workqueue("i915", 0);
751        if (dev_priv->wq == NULL)
752                goto out_err;
753
754        dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
755        if (dev_priv->hotplug.dp_wq == NULL)
756                goto out_free_wq;
757
758        return 0;
759
760out_free_wq:
761        destroy_workqueue(dev_priv->wq);
762out_err:
763        DRM_ERROR("Failed to allocate workqueues.\n");
764
765        return -ENOMEM;
766}
767
768static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
769{
770        destroy_workqueue(dev_priv->hotplug.dp_wq);
771        destroy_workqueue(dev_priv->wq);
772}
773
774/**
775 * i915_driver_init_early - setup state not requiring device access
776 * @dev_priv: device private
777 *
778 * Initialize everything that is a "SW-only" state, that is state not
779 * requiring accessing the device or exposing the driver via kernel internal
780 * or userspace interfaces. Example steps belonging here: lock initialization,
781 * system memory allocation, setting up device specific attributes and
782 * function hooks not requiring accessing the device.
783 */
784static int i915_driver_init_early(struct drm_i915_private *dev_priv,
785                                  const struct pci_device_id *ent)
786{
787        const struct intel_device_info *match_info =
788                (struct intel_device_info *)ent->driver_data;
789        struct intel_device_info *device_info;
790        int ret = 0;
791
792        if (i915_inject_load_failure())
793                return -ENODEV;
794
795        /* Setup the write-once "constant" device info */
796        device_info = mkwrite_device_info(dev_priv);
797        memcpy(device_info, match_info, sizeof(*device_info));
798        device_info->device_id = dev_priv->drm.pdev->device;
799
800        BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
801        device_info->gen_mask = BIT(device_info->gen - 1);
802
803        spin_lock_init(&dev_priv->irq_lock);
804        spin_lock_init(&dev_priv->gpu_error.lock);
805        mutex_init(&dev_priv->backlight_lock);
806        spin_lock_init(&dev_priv->uncore.lock);
807        spin_lock_init(&dev_priv->mm.object_stat_lock);
808        spin_lock_init(&dev_priv->mmio_flip_lock);
809        mutex_init(&dev_priv->sb_lock);
810        mutex_init(&dev_priv->modeset_restore_lock);
811        mutex_init(&dev_priv->av_mutex);
812        mutex_init(&dev_priv->wm.wm_mutex);
813        mutex_init(&dev_priv->pps_mutex);
814
815        i915_memcpy_init_early(dev_priv);
816
817        ret = i915_workqueues_init(dev_priv);
818        if (ret < 0)
819                return ret;
820
821        ret = intel_gvt_init(dev_priv);
822        if (ret < 0)
823                goto err_workqueues;
824
825        /* This must be called before any calls to HAS_PCH_* */
826        intel_detect_pch(&dev_priv->drm);
827
828        intel_pm_setup(&dev_priv->drm);
829        intel_init_dpio(dev_priv);
830        intel_power_domains_init(dev_priv);
831        intel_irq_init(dev_priv);
832        intel_init_display_hooks(dev_priv);
833        intel_init_clock_gating_hooks(dev_priv);
834        intel_init_audio_hooks(dev_priv);
835        i915_gem_load_init(&dev_priv->drm);
836
837        intel_display_crc_init(dev_priv);
838
839        intel_device_info_dump(dev_priv);
840
841        /* Not all pre-production machines fall into this category, only the
842         * very first ones. Almost everything should work, except for maybe
843         * suspend/resume. And we don't implement workarounds that affect only
844         * pre-production machines. */
845        if (IS_HSW_EARLY_SDV(dev_priv))
846                DRM_INFO("This is an early pre-production Haswell machine. "
847                         "It may not be fully functional.\n");
848
849        return 0;
850
851err_workqueues:
852        i915_workqueues_cleanup(dev_priv);
853        return ret;
854}
855
856/**
857 * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
858 * @dev_priv: device private
859 */
860static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
861{
862        i915_gem_load_cleanup(&dev_priv->drm);
863        i915_workqueues_cleanup(dev_priv);
864}
865
866static int i915_mmio_setup(struct drm_device *dev)
867{
868        struct drm_i915_private *dev_priv = to_i915(dev);
869        struct pci_dev *pdev = dev_priv->drm.pdev;
870        int mmio_bar;
871        int mmio_size;
872
873        mmio_bar = IS_GEN2(dev) ? 1 : 0;
874        /*
875         * Before gen4, the registers and the GTT are behind different BARs.
876         * However, from gen4 onwards, the registers and the GTT are shared
877         * in the same BAR, so we want to restrict this ioremap from
878         * clobbering the GTT which we want ioremap_wc instead. Fortunately,
879         * the register BAR remains the same size for all the earlier
880         * generations up to Ironlake.
881         */
882        if (INTEL_INFO(dev)->gen < 5)
883                mmio_size = 512 * 1024;
884        else
885                mmio_size = 2 * 1024 * 1024;
886        dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size);
887        if (dev_priv->regs == NULL) {
888                DRM_ERROR("failed to map registers\n");
889
890                return -EIO;
891        }
892
893        /* Try to make sure MCHBAR is enabled before poking at it */
894        intel_setup_mchbar(dev);
895
896        return 0;
897}
898
899static void i915_mmio_cleanup(struct drm_device *dev)
900{
901        struct drm_i915_private *dev_priv = to_i915(dev);
902        struct pci_dev *pdev = dev_priv->drm.pdev;
903
904        intel_teardown_mchbar(dev);
905        pci_iounmap(pdev, dev_priv->regs);
906}
907
908/**
909 * i915_driver_init_mmio - setup device MMIO
910 * @dev_priv: device private
911 *
912 * Setup minimal device state necessary for MMIO accesses later in the
913 * initialization sequence. The setup here should avoid any other device-wide
914 * side effects or exposing the driver via kernel internal or user space
915 * interfaces.
916 */
917static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
918{
919        struct drm_device *dev = &dev_priv->drm;
920        int ret;
921
922        if (i915_inject_load_failure())
923                return -ENODEV;
924
925        if (i915_get_bridge_dev(dev))
926                return -EIO;
927
928        ret = i915_mmio_setup(dev);
929        if (ret < 0)
930                goto put_bridge;
931
932        intel_uncore_init(dev_priv);
933
934        return 0;
935
936put_bridge:
937        pci_dev_put(dev_priv->bridge_dev);
938
939        return ret;
940}
941
942/**
943 * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
944 * @dev_priv: device private
945 */
946static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
947{
948        struct drm_device *dev = &dev_priv->drm;
949
950        intel_uncore_fini(dev_priv);
951        i915_mmio_cleanup(dev);
952        pci_dev_put(dev_priv->bridge_dev);
953}
954
955static void intel_sanitize_options(struct drm_i915_private *dev_priv)
956{
957        i915.enable_execlists =
958                intel_sanitize_enable_execlists(dev_priv,
959                                                i915.enable_execlists);
960
961        /*
962         * i915.enable_ppgtt is read-only, so do an early pass to validate the
963         * user's requested state against the hardware/driver capabilities.  We
964         * do this now so that we can print out any log messages once rather
965         * than every time we check intel_enable_ppgtt().
966         */
967        i915.enable_ppgtt =
968                intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
969        DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
970
971        i915.semaphores = intel_sanitize_semaphores(dev_priv, i915.semaphores);
972        DRM_DEBUG_DRIVER("use GPU sempahores? %s\n", yesno(i915.semaphores));
973}
974
975/**
976 * i915_driver_init_hw - setup state requiring device access
977 * @dev_priv: device private
978 *
979 * Setup state that requires accessing the device, but doesn't require
980 * exposing the driver via kernel internal or userspace interfaces.
981 */
982static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
983{
984        struct pci_dev *pdev = dev_priv->drm.pdev;
985        struct drm_device *dev = &dev_priv->drm;
986        int ret;
987
988        if (i915_inject_load_failure())
989                return -ENODEV;
990
991        intel_device_info_runtime_init(dev_priv);
992
993        intel_sanitize_options(dev_priv);
994
995        ret = i915_ggtt_probe_hw(dev_priv);
996        if (ret)
997                return ret;
998
999        /* WARNING: Apparently we must kick fbdev drivers before vgacon,
1000         * otherwise the vga fbdev driver falls over. */
1001        ret = i915_kick_out_firmware_fb(dev_priv);
1002        if (ret) {
1003                DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
1004                goto out_ggtt;
1005        }
1006
1007        ret = i915_kick_out_vgacon(dev_priv);
1008        if (ret) {
1009                DRM_ERROR("failed to remove conflicting VGA console\n");
1010                goto out_ggtt;
1011        }
1012
1013        ret = i915_ggtt_init_hw(dev_priv);
1014        if (ret)
1015                return ret;
1016
1017        ret = i915_ggtt_enable_hw(dev_priv);
1018        if (ret) {
1019                DRM_ERROR("failed to enable GGTT\n");
1020                goto out_ggtt;
1021        }
1022
1023        pci_set_master(pdev);
1024
1025        /* overlay on gen2 is broken and can't address above 1G */
1026        if (IS_GEN2(dev)) {
1027                ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
1028                if (ret) {
1029                        DRM_ERROR("failed to set DMA mask\n");
1030
1031                        goto out_ggtt;
1032                }
1033        }
1034
1035        /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1036         * using 32bit addressing, overwriting memory if HWS is located
1037         * above 4GB.
1038         *
1039         * The documentation also mentions an issue with undefined
1040         * behaviour if any general state is accessed within a page above 4GB,
1041         * which also needs to be handled carefully.
1042         */
1043        if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
1044                ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1045
1046                if (ret) {
1047                        DRM_ERROR("failed to set DMA mask\n");
1048
1049                        goto out_ggtt;
1050                }
1051        }
1052
1053        pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
1054                           PM_QOS_DEFAULT_VALUE);
1055
1056        intel_uncore_sanitize(dev_priv);
1057
1058        intel_opregion_setup(dev_priv);
1059
1060        i915_gem_load_init_fences(dev_priv);
1061
1062        /* On the 945G/GM, the chipset reports the MSI capability on the
1063         * integrated graphics even though the support isn't actually there
1064         * according to the published specs.  It doesn't appear to function
1065         * correctly in testing on 945G.
1066         * This may be a side effect of MSI having been made available for PEG
1067         * and the registers being closely associated.
1068         *
1069         * According to chipset errata, on the 965GM, MSI interrupts may
1070         * be lost or delayed, but we use them anyways to avoid
1071         * stuck interrupts on some machines.
1072         */
1073        if (!IS_I945G(dev) && !IS_I945GM(dev)) {
1074                if (pci_enable_msi(pdev) < 0)
1075                        DRM_DEBUG_DRIVER("can't enable MSI");
1076        }
1077
1078        return 0;
1079
1080out_ggtt:
1081        i915_ggtt_cleanup_hw(dev_priv);
1082
1083        return ret;
1084}
1085
1086/**
1087 * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
1088 * @dev_priv: device private
1089 */
1090static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
1091{
1092        struct pci_dev *pdev = dev_priv->drm.pdev;
1093
1094        if (pdev->msi_enabled)
1095                pci_disable_msi(pdev);
1096
1097        pm_qos_remove_request(&dev_priv->pm_qos);
1098        i915_ggtt_cleanup_hw(dev_priv);
1099}
1100
1101/**
1102 * i915_driver_register - register the driver with the rest of the system
1103 * @dev_priv: device private
1104 *
1105 * Perform any steps necessary to make the driver available via kernel
1106 * internal or userspace interfaces.
1107 */
1108static void i915_driver_register(struct drm_i915_private *dev_priv)
1109{
1110        struct drm_device *dev = &dev_priv->drm;
1111
1112        i915_gem_shrinker_init(dev_priv);
1113
1114        /*
1115         * Notify a valid surface after modesetting,
1116         * when running inside a VM.
1117         */
1118        if (intel_vgpu_active(dev_priv))
1119                I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
1120
1121        /* Reveal our presence to userspace */
1122        if (drm_dev_register(dev, 0) == 0) {
1123                i915_debugfs_register(dev_priv);
1124                i915_setup_sysfs(dev_priv);
1125        } else
1126                DRM_ERROR("Failed to register driver for userspace access!\n");
1127
1128        if (INTEL_INFO(dev_priv)->num_pipes) {
1129                /* Must be done after probing outputs */
1130                intel_opregion_register(dev_priv);
1131                acpi_video_register();
1132        }
1133
1134        if (IS_GEN5(dev_priv))
1135                intel_gpu_ips_init(dev_priv);
1136
1137        i915_audio_component_init(dev_priv);
1138
1139        /*
1140         * Some ports require correctly set-up hpd registers for detection to
1141         * work properly (leading to ghost connected connector status), e.g. VGA
1142         * on gm45.  Hence we can only set up the initial fbdev config after hpd
1143         * irqs are fully enabled. We do it last so that the async config
1144         * cannot run before the connectors are registered.
1145         */
1146        intel_fbdev_initial_config_async(dev);
1147}
1148
1149/**
1150 * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
1151 * @dev_priv: device private
1152 */
1153static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1154{
1155        i915_audio_component_cleanup(dev_priv);
1156
1157        intel_gpu_ips_teardown();
1158        acpi_video_unregister();
1159        intel_opregion_unregister(dev_priv);
1160
1161        i915_teardown_sysfs(dev_priv);
1162        i915_debugfs_unregister(dev_priv);
1163        drm_dev_unregister(&dev_priv->drm);
1164
1165        i915_gem_shrinker_cleanup(dev_priv);
1166}
1167
1168/**
1169 * i915_driver_load - setup chip and create an initial config
1170 * @dev: DRM device
1171 * @flags: startup flags
1172 *
1173 * The driver load routine has to do several things:
1174 *   - drive output discovery via intel_modeset_init()
1175 *   - initialize the memory manager
1176 *   - allocate initial config memory
1177 *   - setup the DRM framebuffer with the allocated memory
1178 */
1179int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
1180{
1181        struct drm_i915_private *dev_priv;
1182        int ret;
1183
1184        if (i915.nuclear_pageflip)
1185                driver.driver_features |= DRIVER_ATOMIC;
1186
1187        ret = -ENOMEM;
1188        dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
1189        if (dev_priv)
1190                ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev);
1191        if (ret) {
1192                dev_printk(KERN_ERR, &pdev->dev,
1193                           "[" DRM_NAME ":%s] allocation failed\n", __func__);
1194                kfree(dev_priv);
1195                return ret;
1196        }
1197
1198        dev_priv->drm.pdev = pdev;
1199        dev_priv->drm.dev_private = dev_priv;
1200
1201        ret = pci_enable_device(pdev);
1202        if (ret)
1203                goto out_free_priv;
1204
1205        pci_set_drvdata(pdev, &dev_priv->drm);
1206
1207        ret = i915_driver_init_early(dev_priv, ent);
1208        if (ret < 0)
1209                goto out_pci_disable;
1210
1211        intel_runtime_pm_get(dev_priv);
1212
1213        ret = i915_driver_init_mmio(dev_priv);
1214        if (ret < 0)
1215                goto out_runtime_pm_put;
1216
1217        ret = i915_driver_init_hw(dev_priv);
1218        if (ret < 0)
1219                goto out_cleanup_mmio;
1220
1221        /*
1222         * TODO: move the vblank init and parts of modeset init steps into one
1223         * of the i915_driver_init_/i915_driver_register functions according
1224         * to the role/effect of the given init step.
1225         */
1226        if (INTEL_INFO(dev_priv)->num_pipes) {
1227                ret = drm_vblank_init(&dev_priv->drm,
1228                                      INTEL_INFO(dev_priv)->num_pipes);
1229                if (ret)
1230                        goto out_cleanup_hw;
1231        }
1232
1233        ret = i915_load_modeset_init(&dev_priv->drm);
1234        if (ret < 0)
1235                goto out_cleanup_vblank;
1236
1237        i915_driver_register(dev_priv);
1238
1239        intel_runtime_pm_enable(dev_priv);
1240
1241        /* Everything is in place, we can now relax! */
1242        DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
1243                 driver.name, driver.major, driver.minor, driver.patchlevel,
1244                 driver.date, pci_name(pdev), dev_priv->drm.primary->index);
1245
1246        intel_runtime_pm_put(dev_priv);
1247
1248        return 0;
1249
1250out_cleanup_vblank:
1251        drm_vblank_cleanup(&dev_priv->drm);
1252out_cleanup_hw:
1253        i915_driver_cleanup_hw(dev_priv);
1254out_cleanup_mmio:
1255        i915_driver_cleanup_mmio(dev_priv);
1256out_runtime_pm_put:
1257        intel_runtime_pm_put(dev_priv);
1258        i915_driver_cleanup_early(dev_priv);
1259out_pci_disable:
1260        pci_disable_device(pdev);
1261out_free_priv:
1262        i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
1263        drm_dev_unref(&dev_priv->drm);
1264        return ret;
1265}
1266
1267void i915_driver_unload(struct drm_device *dev)
1268{
1269        struct drm_i915_private *dev_priv = to_i915(dev);
1270        struct pci_dev *pdev = dev_priv->drm.pdev;
1271
1272        intel_fbdev_fini(dev);
1273
1274        if (i915_gem_suspend(dev))
1275                DRM_ERROR("failed to idle hardware; continuing to unload!\n");
1276
1277        intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
1278
1279        i915_driver_unregister(dev_priv);
1280
1281        drm_vblank_cleanup(dev);
1282
1283        intel_modeset_cleanup(dev);
1284
1285        /*
1286         * free the memory space allocated for the child device
1287         * config parsed from VBT
1288         */
1289        if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
1290                kfree(dev_priv->vbt.child_dev);
1291                dev_priv->vbt.child_dev = NULL;
1292                dev_priv->vbt.child_dev_num = 0;
1293        }
1294        kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
1295        dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
1296        kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
1297        dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
1298
1299        vga_switcheroo_unregister_client(pdev);
1300        vga_client_register(pdev, NULL, NULL, NULL);
1301
1302        intel_csr_ucode_fini(dev_priv);
1303
1304        /* Free error state after interrupts are fully disabled. */
1305        cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
1306        i915_destroy_error_state(dev);
1307
1308        /* Flush any outstanding unpin_work. */
1309        drain_workqueue(dev_priv->wq);
1310
1311        intel_guc_fini(dev);
1312        i915_gem_fini(dev);
1313        intel_fbc_cleanup_cfb(dev_priv);
1314
1315        intel_power_domains_fini(dev_priv);
1316
1317        i915_driver_cleanup_hw(dev_priv);
1318        i915_driver_cleanup_mmio(dev_priv);
1319
1320        intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
1321
1322        i915_driver_cleanup_early(dev_priv);
1323}
1324
1325static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
1326{
1327        int ret;
1328
1329        ret = i915_gem_open(dev, file);
1330        if (ret)
1331                return ret;
1332
1333        return 0;
1334}
1335
1336/**
1337 * i915_driver_lastclose - clean up after all DRM clients have exited
1338 * @dev: DRM device
1339 *
1340 * Take care of cleaning up after all DRM clients have exited.  In the
1341 * mode setting case, we want to restore the kernel's initial mode (just
1342 * in case the last client left us in a bad state).
1343 *
1344 * Additionally, in the non-mode setting case, we'll tear down the GTT
1345 * and DMA structures, since the kernel won't be using them, and clea
1346 * up any GEM state.
1347 */
1348static void i915_driver_lastclose(struct drm_device *dev)
1349{
1350        intel_fbdev_restore_mode(dev);
1351        vga_switcheroo_process_delayed_switch();
1352}
1353
1354static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
1355{
1356        mutex_lock(&dev->struct_mutex);
1357        i915_gem_context_close(dev, file);
1358        i915_gem_release(dev, file);
1359        mutex_unlock(&dev->struct_mutex);
1360}
1361
1362static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1363{
1364        struct drm_i915_file_private *file_priv = file->driver_priv;
1365
1366        kfree(file_priv);
1367}
1368
1369static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
1370{
1371        struct drm_device *dev = &dev_priv->drm;
1372        struct intel_encoder *encoder;
1373
1374        drm_modeset_lock_all(dev);
1375        for_each_intel_encoder(dev, encoder)
1376                if (encoder->suspend)
1377                        encoder->suspend(encoder);
1378        drm_modeset_unlock_all(dev);
1379}
1380
1381static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1382                              bool rpm_resume);
1383static int vlv_suspend_complete(struct drm_i915_private *dev_priv);
1384
1385static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1386{
1387#if IS_ENABLED(CONFIG_ACPI_SLEEP)
1388        if (acpi_target_system_state() < ACPI_STATE_S3)
1389                return true;
1390#endif
1391        return false;
1392}
1393
1394static int i915_drm_suspend(struct drm_device *dev)
1395{
1396        struct drm_i915_private *dev_priv = to_i915(dev);
1397        struct pci_dev *pdev = dev_priv->drm.pdev;
1398        pci_power_t opregion_target_state;
1399        int error;
1400
1401        /* ignore lid events during suspend */
1402        mutex_lock(&dev_priv->modeset_restore_lock);
1403        dev_priv->modeset_restore = MODESET_SUSPENDED;
1404        mutex_unlock(&dev_priv->modeset_restore_lock);
1405
1406        disable_rpm_wakeref_asserts(dev_priv);
1407
1408        /* We do a lot of poking in a lot of registers, make sure they work
1409         * properly. */
1410        intel_display_set_init_power(dev_priv, true);
1411
1412        drm_kms_helper_poll_disable(dev);
1413
1414        pci_save_state(pdev);
1415
1416        error = i915_gem_suspend(dev);
1417        if (error) {
1418                dev_err(&pdev->dev,
1419                        "GEM idle failed, resume might fail\n");
1420                goto out;
1421        }
1422
1423        intel_guc_suspend(dev);
1424
1425        intel_display_suspend(dev);
1426
1427        intel_dp_mst_suspend(dev);
1428
1429        intel_runtime_pm_disable_interrupts(dev_priv);
1430        intel_hpd_cancel_work(dev_priv);
1431
1432        intel_suspend_encoders(dev_priv);
1433
1434        intel_suspend_hw(dev);
1435
1436        i915_gem_suspend_gtt_mappings(dev);
1437
1438        i915_save_state(dev);
1439
1440        opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
1441        intel_opregion_notify_adapter(dev_priv, opregion_target_state);
1442
1443        intel_uncore_forcewake_reset(dev_priv, false);
1444        intel_opregion_unregister(dev_priv);
1445
1446        intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
1447
1448        dev_priv->suspend_count++;
1449
1450        intel_csr_ucode_suspend(dev_priv);
1451
1452out:
1453        enable_rpm_wakeref_asserts(dev_priv);
1454
1455        return error;
1456}
1457
1458static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1459{
1460        struct drm_i915_private *dev_priv = to_i915(dev);
1461        struct pci_dev *pdev = dev_priv->drm.pdev;
1462        bool fw_csr;
1463        int ret;
1464
1465        disable_rpm_wakeref_asserts(dev_priv);
1466
1467        intel_display_set_init_power(dev_priv, false);
1468
1469        fw_csr = !IS_BROXTON(dev_priv) &&
1470                suspend_to_idle(dev_priv) && dev_priv->csr.dmc_payload;
1471        /*
1472         * In case of firmware assisted context save/restore don't manually
1473         * deinit the power domains. This also means the CSR/DMC firmware will
1474         * stay active, it will power down any HW resources as required and
1475         * also enable deeper system power states that would be blocked if the
1476         * firmware was inactive.
1477         */
1478        if (!fw_csr)
1479                intel_power_domains_suspend(dev_priv);
1480
1481        ret = 0;
1482        if (IS_BROXTON(dev_priv))
1483                bxt_enable_dc9(dev_priv);
1484        else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
1485                hsw_enable_pc8(dev_priv);
1486        else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1487                ret = vlv_suspend_complete(dev_priv);
1488
1489        if (ret) {
1490                DRM_ERROR("Suspend complete failed: %d\n", ret);
1491                if (!fw_csr)
1492                        intel_power_domains_init_hw(dev_priv, true);
1493
1494                goto out;
1495        }
1496
1497        pci_disable_device(pdev);
1498        /*
1499         * During hibernation on some platforms the BIOS may try to access
1500         * the device even though it's already in D3 and hang the machine. So
1501         * leave the device in D0 on those platforms and hope the BIOS will
1502         * power down the device properly. The issue was seen on multiple old
1503         * GENs with different BIOS vendors, so having an explicit blacklist
1504         * is inpractical; apply the workaround on everything pre GEN6. The
1505         * platforms where the issue was seen:
1506         * Lenovo Thinkpad X301, X61s, X60, T60, X41
1507         * Fujitsu FSC S7110
1508         * Acer Aspire 1830T
1509         */
1510        if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
1511                pci_set_power_state(pdev, PCI_D3hot);
1512
1513        dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
1514
1515out:
1516        enable_rpm_wakeref_asserts(dev_priv);
1517
1518        return ret;
1519}
1520
1521int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
1522{
1523        int error;
1524
1525        if (!dev) {
1526                DRM_ERROR("dev: %p\n", dev);
1527                DRM_ERROR("DRM not initialized, aborting suspend.\n");
1528                return -ENODEV;
1529        }
1530
1531        if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
1532                         state.event != PM_EVENT_FREEZE))
1533                return -EINVAL;
1534
1535        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1536                return 0;
1537
1538        error = i915_drm_suspend(dev);
1539        if (error)
1540                return error;
1541
1542        return i915_drm_suspend_late(dev, false);
1543}
1544
1545static int i915_drm_resume(struct drm_device *dev)
1546{
1547        struct drm_i915_private *dev_priv = to_i915(dev);
1548        int ret;
1549
1550        disable_rpm_wakeref_asserts(dev_priv);
1551        intel_sanitize_gt_powersave(dev_priv);
1552
1553        ret = i915_ggtt_enable_hw(dev_priv);
1554        if (ret)
1555                DRM_ERROR("failed to re-enable GGTT\n");
1556
1557        intel_csr_ucode_resume(dev_priv);
1558
1559        i915_gem_resume(dev);
1560
1561        i915_restore_state(dev);
1562        intel_pps_unlock_regs_wa(dev_priv);
1563        intel_opregion_setup(dev_priv);
1564
1565        intel_init_pch_refclk(dev);
1566        drm_mode_config_reset(dev);
1567
1568        /*
1569         * Interrupts have to be enabled before any batches are run. If not the
1570         * GPU will hang. i915_gem_init_hw() will initiate batches to
1571         * update/restore the context.
1572         *
1573         * Modeset enabling in intel_modeset_init_hw() also needs working
1574         * interrupts.
1575         */
1576        intel_runtime_pm_enable_interrupts(dev_priv);
1577
1578        mutex_lock(&dev->struct_mutex);
1579        if (i915_gem_init_hw(dev)) {
1580                DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
1581                i915_gem_set_wedged(dev_priv);
1582        }
1583        mutex_unlock(&dev->struct_mutex);
1584
1585        intel_guc_resume(dev);
1586
1587        intel_modeset_init_hw(dev);
1588
1589        spin_lock_irq(&dev_priv->irq_lock);
1590        if (dev_priv->display.hpd_irq_setup)
1591                dev_priv->display.hpd_irq_setup(dev_priv);
1592        spin_unlock_irq(&dev_priv->irq_lock);
1593
1594        intel_dp_mst_resume(dev);
1595
1596        intel_display_resume(dev);
1597
1598        /*
1599         * ... but also need to make sure that hotplug processing
1600         * doesn't cause havoc. Like in the driver load code we don't
1601         * bother with the tiny race here where we might loose hotplug
1602         * notifications.
1603         * */
1604        intel_hpd_init(dev_priv);
1605        /* Config may have changed between suspend and resume */
1606        drm_helper_hpd_irq_event(dev);
1607
1608        intel_opregion_register(dev_priv);
1609
1610        intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
1611
1612        mutex_lock(&dev_priv->modeset_restore_lock);
1613        dev_priv->modeset_restore = MODESET_DONE;
1614        mutex_unlock(&dev_priv->modeset_restore_lock);
1615
1616        intel_opregion_notify_adapter(dev_priv, PCI_D0);
1617
1618        intel_autoenable_gt_powersave(dev_priv);
1619        drm_kms_helper_poll_enable(dev);
1620
1621        enable_rpm_wakeref_asserts(dev_priv);
1622
1623        return 0;
1624}
1625
1626static int i915_drm_resume_early(struct drm_device *dev)
1627{
1628        struct drm_i915_private *dev_priv = to_i915(dev);
1629        struct pci_dev *pdev = dev_priv->drm.pdev;
1630        int ret;
1631
1632        /*
1633         * We have a resume ordering issue with the snd-hda driver also
1634         * requiring our device to be power up. Due to the lack of a
1635         * parent/child relationship we currently solve this with an early
1636         * resume hook.
1637         *
1638         * FIXME: This should be solved with a special hdmi sink device or
1639         * similar so that power domains can be employed.
1640         */
1641
1642        /*
1643         * Note that we need to set the power state explicitly, since we
1644         * powered off the device during freeze and the PCI core won't power
1645         * it back up for us during thaw. Powering off the device during
1646         * freeze is not a hard requirement though, and during the
1647         * suspend/resume phases the PCI core makes sure we get here with the
1648         * device powered on. So in case we change our freeze logic and keep
1649         * the device powered we can also remove the following set power state
1650         * call.
1651         */
1652        ret = pci_set_power_state(pdev, PCI_D0);
1653        if (ret) {
1654                DRM_ERROR("failed to set PCI D0 power state (%d)\n", ret);
1655                goto out;
1656        }
1657
1658        /*
1659         * Note that pci_enable_device() first enables any parent bridge
1660         * device and only then sets the power state for this device. The
1661         * bridge enabling is a nop though, since bridge devices are resumed
1662         * first. The order of enabling power and enabling the device is
1663         * imposed by the PCI core as described above, so here we preserve the
1664         * same order for the freeze/thaw phases.
1665         *
1666         * TODO: eventually we should remove pci_disable_device() /
1667         * pci_enable_enable_device() from suspend/resume. Due to how they
1668         * depend on the device enable refcount we can't anyway depend on them
1669         * disabling/enabling the device.
1670         */
1671        if (pci_enable_device(pdev)) {
1672                ret = -EIO;
1673                goto out;
1674        }
1675
1676        pci_set_master(pdev);
1677
1678        disable_rpm_wakeref_asserts(dev_priv);
1679
1680        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1681                ret = vlv_resume_prepare(dev_priv, false);
1682        if (ret)
1683                DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
1684                          ret);
1685
1686        intel_uncore_early_sanitize(dev_priv, true);
1687
1688        if (IS_BROXTON(dev_priv)) {
1689                if (!dev_priv->suspended_to_idle)
1690                        gen9_sanitize_dc_state(dev_priv);
1691                bxt_disable_dc9(dev_priv);
1692        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1693                hsw_disable_pc8(dev_priv);
1694        }
1695
1696        intel_uncore_sanitize(dev_priv);
1697
1698        if (IS_BROXTON(dev_priv) ||
1699            !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
1700                intel_power_domains_init_hw(dev_priv, true);
1701
1702        enable_rpm_wakeref_asserts(dev_priv);
1703
1704out:
1705        dev_priv->suspended_to_idle = false;
1706
1707        return ret;
1708}
1709
1710int i915_resume_switcheroo(struct drm_device *dev)
1711{
1712        int ret;
1713
1714        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1715                return 0;
1716
1717        ret = i915_drm_resume_early(dev);
1718        if (ret)
1719                return ret;
1720
1721        return i915_drm_resume(dev);
1722}
1723
1724/**
1725 * i915_reset - reset chip after a hang
1726 * @dev: drm device to reset
1727 *
1728 * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
1729 * on failure.
1730 *
1731 * Caller must hold the struct_mutex.
1732 *
1733 * Procedure is fairly simple:
1734 *   - reset the chip using the reset reg
1735 *   - re-init context state
1736 *   - re-init hardware status page
1737 *   - re-init ring buffer
1738 *   - re-init interrupt state
1739 *   - re-init display
1740 */
1741void i915_reset(struct drm_i915_private *dev_priv)
1742{
1743        struct drm_device *dev = &dev_priv->drm;
1744        struct i915_gpu_error *error = &dev_priv->gpu_error;
1745        int ret;
1746
1747        lockdep_assert_held(&dev->struct_mutex);
1748
1749        if (!test_and_clear_bit(I915_RESET_IN_PROGRESS, &error->flags))
1750                return;
1751
1752        /* Clear any previous failed attempts at recovery. Time to try again. */
1753        __clear_bit(I915_WEDGED, &error->flags);
1754        error->reset_count++;
1755
1756        pr_notice("drm/i915: Resetting chip after gpu hang\n");
1757        ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
1758        if (ret) {
1759                if (ret != -ENODEV)
1760                        DRM_ERROR("Failed to reset chip: %i\n", ret);
1761                else
1762                        DRM_DEBUG_DRIVER("GPU reset disabled\n");
1763                goto error;
1764        }
1765
1766        i915_gem_reset(dev_priv);
1767        intel_overlay_reset(dev_priv);
1768
1769        /* Ok, now get things going again... */
1770
1771        /*
1772         * Everything depends on having the GTT running, so we need to start
1773         * there.  Fortunately we don't need to do this unless we reset the
1774         * chip at a PCI level.
1775         *
1776         * Next we need to restore the context, but we don't use those
1777         * yet either...
1778         *
1779         * Ring buffer needs to be re-initialized in the KMS case, or if X
1780         * was running at the time of the reset (i.e. we weren't VT
1781         * switched away).
1782         */
1783        ret = i915_gem_init_hw(dev);
1784        if (ret) {
1785                DRM_ERROR("Failed hw init on reset %d\n", ret);
1786                goto error;
1787        }
1788
1789wakeup:
1790        wake_up_bit(&error->flags, I915_RESET_IN_PROGRESS);
1791        return;
1792
1793error:
1794        i915_gem_set_wedged(dev_priv);
1795        goto wakeup;
1796}
1797
1798static int i915_pm_suspend(struct device *kdev)
1799{
1800        struct pci_dev *pdev = to_pci_dev(kdev);
1801        struct drm_device *dev = pci_get_drvdata(pdev);
1802
1803        if (!dev) {
1804                dev_err(kdev, "DRM not initialized, aborting suspend.\n");
1805                return -ENODEV;
1806        }
1807
1808        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1809                return 0;
1810
1811        return i915_drm_suspend(dev);
1812}
1813
1814static int i915_pm_suspend_late(struct device *kdev)
1815{
1816        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1817
1818        /*
1819         * We have a suspend ordering issue with the snd-hda driver also
1820         * requiring our device to be power up. Due to the lack of a
1821         * parent/child relationship we currently solve this with an late
1822         * suspend hook.
1823         *
1824         * FIXME: This should be solved with a special hdmi sink device or
1825         * similar so that power domains can be employed.
1826         */
1827        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1828                return 0;
1829
1830        return i915_drm_suspend_late(dev, false);
1831}
1832
1833static int i915_pm_poweroff_late(struct device *kdev)
1834{
1835        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1836
1837        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1838                return 0;
1839
1840        return i915_drm_suspend_late(dev, true);
1841}
1842
1843static int i915_pm_resume_early(struct device *kdev)
1844{
1845        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1846
1847        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1848                return 0;
1849
1850        return i915_drm_resume_early(dev);
1851}
1852
1853static int i915_pm_resume(struct device *kdev)
1854{
1855        struct drm_device *dev = &kdev_to_i915(kdev)->drm;
1856
1857        if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1858                return 0;
1859
1860        return i915_drm_resume(dev);
1861}
1862
1863/* freeze: before creating the hibernation_image */
1864static int i915_pm_freeze(struct device *kdev)
1865{
1866        int ret;
1867
1868        ret = i915_pm_suspend(kdev);
1869        if (ret)
1870                return ret;
1871
1872        ret = i915_gem_freeze(kdev_to_i915(kdev));
1873        if (ret)
1874                return ret;
1875
1876        return 0;
1877}
1878
1879static int i915_pm_freeze_late(struct device *kdev)
1880{
1881        int ret;
1882
1883        ret = i915_pm_suspend_late(kdev);
1884        if (ret)
1885                return ret;
1886
1887        ret = i915_gem_freeze_late(kdev_to_i915(kdev));
1888        if (ret)
1889                return ret;
1890
1891        return 0;
1892}
1893
1894/* thaw: called after creating the hibernation image, but before turning off. */
1895static int i915_pm_thaw_early(struct device *kdev)
1896{
1897        return i915_pm_resume_early(kdev);
1898}
1899
1900static int i915_pm_thaw(struct device *kdev)
1901{
1902        return i915_pm_resume(kdev);
1903}
1904
1905/* restore: called after loading the hibernation image. */
1906static int i915_pm_restore_early(struct device *kdev)
1907{
1908        return i915_pm_resume_early(kdev);
1909}
1910
1911static int i915_pm_restore(struct device *kdev)
1912{
1913        return i915_pm_resume(kdev);
1914}
1915
1916/*
1917 * Save all Gunit registers that may be lost after a D3 and a subsequent
1918 * S0i[R123] transition. The list of registers needing a save/restore is
1919 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
1920 * registers in the following way:
1921 * - Driver: saved/restored by the driver
1922 * - Punit : saved/restored by the Punit firmware
1923 * - No, w/o marking: no need to save/restore, since the register is R/O or
1924 *                    used internally by the HW in a way that doesn't depend
1925 *                    keeping the content across a suspend/resume.
1926 * - Debug : used for debugging
1927 *
1928 * We save/restore all registers marked with 'Driver', with the following
1929 * exceptions:
1930 * - Registers out of use, including also registers marked with 'Debug'.
1931 *   These have no effect on the driver's operation, so we don't save/restore
1932 *   them to reduce the overhead.
1933 * - Registers that are fully setup by an initialization function called from
1934 *   the resume path. For example many clock gating and RPS/RC6 registers.
1935 * - Registers that provide the right functionality with their reset defaults.
1936 *
1937 * TODO: Except for registers that based on the above 3 criteria can be safely
1938 * ignored, we save/restore all others, practically treating the HW context as
1939 * a black-box for the driver. Further investigation is needed to reduce the
1940 * saved/restored registers even further, by following the same 3 criteria.
1941 */
1942static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
1943{
1944        struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
1945        int i;
1946
1947        /* GAM 0x4000-0x4770 */
1948        s->wr_watermark         = I915_READ(GEN7_WR_WATERMARK);
1949        s->gfx_prio_ctrl        = I915_READ(GEN7_GFX_PRIO_CTRL);
1950        s->arb_mode             = I915_READ(ARB_MODE);
1951        s->gfx_pend_tlb0        = I915_READ(GEN7_GFX_PEND_TLB0);
1952        s->gfx_pend_tlb1        = I915_READ(GEN7_GFX_PEND_TLB1);
1953
1954        for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
1955                s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
1956
1957        s->media_max_req_count  = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
1958        s->gfx_max_req_count    = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
1959
1960        s->render_hwsp          = I915_READ(RENDER_HWS_PGA_GEN7);
1961        s->ecochk               = I915_READ(GAM_ECOCHK);
1962        s->bsd_hwsp             = I915_READ(BSD_HWS_PGA_GEN7);
1963        s->blt_hwsp             = I915_READ(BLT_HWS_PGA_GEN7);
1964
1965        s->tlb_rd_addr          = I915_READ(GEN7_TLB_RD_ADDR);
1966
1967        /* MBC 0x9024-0x91D0, 0x8500 */
1968        s->g3dctl               = I915_READ(VLV_G3DCTL);
1969        s->gsckgctl             = I915_READ(VLV_GSCKGCTL);
1970        s->mbctl                = I915_READ(GEN6_MBCTL);
1971
1972        /* GCP 0x9400-0x9424, 0x8100-0x810C */
1973        s->ucgctl1              = I915_READ(GEN6_UCGCTL1);
1974        s->ucgctl3              = I915_READ(GEN6_UCGCTL3);
1975        s->rcgctl1              = I915_READ(GEN6_RCGCTL1);
1976        s->rcgctl2              = I915_READ(GEN6_RCGCTL2);
1977        s->rstctl               = I915_READ(GEN6_RSTCTL);
1978        s->misccpctl            = I915_READ(GEN7_MISCCPCTL);
1979
1980        /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
1981        s->gfxpause             = I915_READ(GEN6_GFXPAUSE);
1982        s->rpdeuhwtc            = I915_READ(GEN6_RPDEUHWTC);
1983        s->rpdeuc               = I915_READ(GEN6_RPDEUC);
1984        s->ecobus               = I915_READ(ECOBUS);
1985        s->pwrdwnupctl          = I915_READ(VLV_PWRDWNUPCTL);
1986        s->rp_down_timeout      = I915_READ(GEN6_RP_DOWN_TIMEOUT);
1987        s->rp_deucsw            = I915_READ(GEN6_RPDEUCSW);
1988        s->rcubmabdtmr          = I915_READ(GEN6_RCUBMABDTMR);
1989        s->rcedata              = I915_READ(VLV_RCEDATA);
1990        s->spare2gh             = I915_READ(VLV_SPAREG2H);
1991
1992        /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
1993        s->gt_imr               = I915_READ(GTIMR);
1994        s->gt_ier               = I915_READ(GTIER);
1995        s->pm_imr               = I915_READ(GEN6_PMIMR);
1996        s->pm_ier               = I915_READ(GEN6_PMIER);
1997
1998        for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
1999                s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
2000
2001        /* GT SA CZ domain, 0x100000-0x138124 */
2002        s->tilectl              = I915_READ(TILECTL);
2003        s->gt_fifoctl           = I915_READ(GTFIFOCTL);
2004        s->gtlc_wake_ctrl       = I915_READ(VLV_GTLC_WAKE_CTRL);
2005        s->gtlc_survive         = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2006        s->pmwgicz              = I915_READ(VLV_PMWGICZ);
2007
2008        /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2009        s->gu_ctl0              = I915_READ(VLV_GU_CTL0);
2010        s->gu_ctl1              = I915_READ(VLV_GU_CTL1);
2011        s->pcbr                 = I915_READ(VLV_PCBR);
2012        s->clock_gate_dis2      = I915_READ(VLV_GUNIT_CLOCK_GATE2);
2013
2014        /*
2015         * Not saving any of:
2016         * DFT,         0x9800-0x9EC0
2017         * SARB,        0xB000-0xB1FC
2018         * GAC,         0x5208-0x524C, 0x14000-0x14C000
2019         * PCI CFG
2020         */
2021}
2022
2023static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
2024{
2025        struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
2026        u32 val;
2027        int i;
2028
2029        /* GAM 0x4000-0x4770 */
2030        I915_WRITE(GEN7_WR_WATERMARK,   s->wr_watermark);
2031        I915_WRITE(GEN7_GFX_PRIO_CTRL,  s->gfx_prio_ctrl);
2032        I915_WRITE(ARB_MODE,            s->arb_mode | (0xffff << 16));
2033        I915_WRITE(GEN7_GFX_PEND_TLB0,  s->gfx_pend_tlb0);
2034        I915_WRITE(GEN7_GFX_PEND_TLB1,  s->gfx_pend_tlb1);
2035
2036        for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
2037                I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
2038
2039        I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
2040        I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
2041
2042        I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
2043        I915_WRITE(GAM_ECOCHK,          s->ecochk);
2044        I915_WRITE(BSD_HWS_PGA_GEN7,    s->bsd_hwsp);
2045        I915_WRITE(BLT_HWS_PGA_GEN7,    s->blt_hwsp);
2046
2047        I915_WRITE(GEN7_TLB_RD_ADDR,    s->tlb_rd_addr);
2048
2049        /* MBC 0x9024-0x91D0, 0x8500 */
2050        I915_WRITE(VLV_G3DCTL,          s->g3dctl);
2051        I915_WRITE(VLV_GSCKGCTL,        s->gsckgctl);
2052        I915_WRITE(GEN6_MBCTL,          s->mbctl);
2053
2054        /* GCP 0x9400-0x9424, 0x8100-0x810C */
2055        I915_WRITE(GEN6_UCGCTL1,        s->ucgctl1);
2056        I915_WRITE(GEN6_UCGCTL3,        s->ucgctl3);
2057        I915_WRITE(GEN6_RCGCTL1,        s->rcgctl1);
2058        I915_WRITE(GEN6_RCGCTL2,        s->rcgctl2);
2059        I915_WRITE(GEN6_RSTCTL,         s->rstctl);
2060        I915_WRITE(GEN7_MISCCPCTL,      s->misccpctl);
2061
2062        /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
2063        I915_WRITE(GEN6_GFXPAUSE,       s->gfxpause);
2064        I915_WRITE(GEN6_RPDEUHWTC,      s->rpdeuhwtc);
2065        I915_WRITE(GEN6_RPDEUC,         s->rpdeuc);
2066        I915_WRITE(ECOBUS,              s->ecobus);
2067        I915_WRITE(VLV_PWRDWNUPCTL,     s->pwrdwnupctl);
2068        I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
2069        I915_WRITE(GEN6_RPDEUCSW,       s->rp_deucsw);
2070        I915_WRITE(GEN6_RCUBMABDTMR,    s->rcubmabdtmr);
2071        I915_WRITE(VLV_RCEDATA,         s->rcedata);
2072        I915_WRITE(VLV_SPAREG2H,        s->spare2gh);
2073
2074        /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
2075        I915_WRITE(GTIMR,               s->gt_imr);
2076        I915_WRITE(GTIER,               s->gt_ier);
2077        I915_WRITE(GEN6_PMIMR,          s->pm_imr);
2078        I915_WRITE(GEN6_PMIER,          s->pm_ier);
2079
2080        for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
2081                I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
2082
2083        /* GT SA CZ domain, 0x100000-0x138124 */
2084        I915_WRITE(TILECTL,                     s->tilectl);
2085        I915_WRITE(GTFIFOCTL,                   s->gt_fifoctl);
2086        /*
2087         * Preserve the GT allow wake and GFX force clock bit, they are not
2088         * be restored, as they are used to control the s0ix suspend/resume
2089         * sequence by the caller.
2090         */
2091        val = I915_READ(VLV_GTLC_WAKE_CTRL);
2092        val &= VLV_GTLC_ALLOWWAKEREQ;
2093        val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
2094        I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2095
2096        val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2097        val &= VLV_GFX_CLK_FORCE_ON_BIT;
2098        val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
2099        I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2100
2101        I915_WRITE(VLV_PMWGICZ,                 s->pmwgicz);
2102
2103        /* Gunit-Display CZ domain, 0x182028-0x1821CF */
2104        I915_WRITE(VLV_GU_CTL0,                 s->gu_ctl0);
2105        I915_WRITE(VLV_GU_CTL1,                 s->gu_ctl1);
2106        I915_WRITE(VLV_PCBR,                    s->pcbr);
2107        I915_WRITE(VLV_GUNIT_CLOCK_GATE2,       s->clock_gate_dis2);
2108}
2109
2110int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
2111{
2112        u32 val;
2113        int err;
2114
2115        val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
2116        val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
2117        if (force_on)
2118                val |= VLV_GFX_CLK_FORCE_ON_BIT;
2119        I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
2120
2121        if (!force_on)
2122                return 0;
2123
2124        err = intel_wait_for_register(dev_priv,
2125                                      VLV_GTLC_SURVIVABILITY_REG,
2126                                      VLV_GFX_CLK_STATUS_BIT,
2127                                      VLV_GFX_CLK_STATUS_BIT,
2128                                      20);
2129        if (err)
2130                DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
2131                          I915_READ(VLV_GTLC_SURVIVABILITY_REG));
2132
2133        return err;
2134}
2135
2136static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
2137{
2138        u32 val;
2139        int err = 0;
2140
2141        val = I915_READ(VLV_GTLC_WAKE_CTRL);
2142        val &= ~VLV_GTLC_ALLOWWAKEREQ;
2143        if (allow)
2144                val |= VLV_GTLC_ALLOWWAKEREQ;
2145        I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
2146        POSTING_READ(VLV_GTLC_WAKE_CTRL);
2147
2148        err = intel_wait_for_register(dev_priv,
2149                                      VLV_GTLC_PW_STATUS,
2150                                      VLV_GTLC_ALLOWWAKEACK,
2151                                      allow,
2152                                      1);
2153        if (err)
2154                DRM_ERROR("timeout disabling GT waking\n");
2155
2156        return err;
2157}
2158
2159static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
2160                                 bool wait_for_on)
2161{
2162        u32 mask;
2163        u32 val;
2164        int err;
2165
2166        mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
2167        val = wait_for_on ? mask : 0;
2168        if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
2169                return 0;
2170
2171        DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
2172                      onoff(wait_for_on),
2173                      I915_READ(VLV_GTLC_PW_STATUS));
2174
2175        /*
2176         * RC6 transitioning can be delayed up to 2 msec (see
2177         * valleyview_enable_rps), use 3 msec for safety.
2178         */
2179        err = intel_wait_for_register(dev_priv,
2180                                      VLV_GTLC_PW_STATUS, mask, val,
2181                                      3);
2182        if (err)
2183                DRM_ERROR("timeout waiting for GT wells to go %s\n",
2184                          onoff(wait_for_on));
2185
2186        return err;
2187}
2188
2189static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
2190{
2191        if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
2192                return;
2193
2194        DRM_DEBUG_DRIVER("GT register access while GT waking disabled\n");
2195        I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
2196}
2197
2198static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
2199{
2200        u32 mask;
2201        int err;
2202
2203        /*
2204         * Bspec defines the following GT well on flags as debug only, so
2205         * don't treat them as hard failures.
2206         */
2207        (void)vlv_wait_for_gt_wells(dev_priv, false);
2208
2209        mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
2210        WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
2211
2212        vlv_check_no_gt_access(dev_priv);
2213
2214        err = vlv_force_gfx_clock(dev_priv, true);
2215        if (err)
2216                goto err1;
2217
2218        err = vlv_allow_gt_wake(dev_priv, false);
2219        if (err)
2220                goto err2;
2221
2222        if (!IS_CHERRYVIEW(dev_priv))
2223                vlv_save_gunit_s0ix_state(dev_priv);
2224
2225        err = vlv_force_gfx_clock(dev_priv, false);
2226        if (err)
2227                goto err2;
2228
2229        return 0;
2230
2231err2:
2232        /* For safety always re-enable waking and disable gfx clock forcing */
2233        vlv_allow_gt_wake(dev_priv, true);
2234err1:
2235        vlv_force_gfx_clock(dev_priv, false);
2236
2237        return err;
2238}
2239
2240static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
2241                                bool rpm_resume)
2242{
2243        struct drm_device *dev = &dev_priv->drm;
2244        int err;
2245        int ret;
2246
2247        /*
2248         * If any of the steps fail just try to continue, that's the best we
2249         * can do at this point. Return the first error code (which will also
2250         * leave RPM permanently disabled).
2251         */
2252        ret = vlv_force_gfx_clock(dev_priv, true);
2253
2254        if (!IS_CHERRYVIEW(dev_priv))
2255                vlv_restore_gunit_s0ix_state(dev_priv);
2256
2257        err = vlv_allow_gt_wake(dev_priv, true);
2258        if (!ret)
2259                ret = err;
2260
2261        err = vlv_force_gfx_clock(dev_priv, false);
2262        if (!ret)
2263                ret = err;
2264
2265        vlv_check_no_gt_access(dev_priv);
2266
2267        if (rpm_resume) {
2268                intel_init_clock_gating(dev);
2269                i915_gem_restore_fences(dev);
2270        }
2271
2272        return ret;
2273}
2274
2275static int intel_runtime_suspend(struct device *kdev)
2276{
2277        struct pci_dev *pdev = to_pci_dev(kdev);
2278        struct drm_device *dev = pci_get_drvdata(pdev);
2279        struct drm_i915_private *dev_priv = to_i915(dev);
2280        int ret;
2281
2282        if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
2283                return -ENODEV;
2284
2285        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2286                return -ENODEV;
2287
2288        DRM_DEBUG_KMS("Suspending device\n");
2289
2290        /*
2291         * We could deadlock here in case another thread holding struct_mutex
2292         * calls RPM suspend concurrently, since the RPM suspend will wait
2293         * first for this RPM suspend to finish. In this case the concurrent
2294         * RPM resume will be followed by its RPM suspend counterpart. Still
2295         * for consistency return -EAGAIN, which will reschedule this suspend.
2296         */
2297        if (!mutex_trylock(&dev->struct_mutex)) {
2298                DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
2299                /*
2300                 * Bump the expiration timestamp, otherwise the suspend won't
2301                 * be rescheduled.
2302                 */
2303                pm_runtime_mark_last_busy(kdev);
2304
2305                return -EAGAIN;
2306        }
2307
2308        disable_rpm_wakeref_asserts(dev_priv);
2309
2310        /*
2311         * We are safe here against re-faults, since the fault handler takes
2312         * an RPM reference.
2313         */
2314        i915_gem_release_all_mmaps(dev_priv);
2315        mutex_unlock(&dev->struct_mutex);
2316
2317        intel_guc_suspend(dev);
2318
2319        intel_runtime_pm_disable_interrupts(dev_priv);
2320
2321        ret = 0;
2322        if (IS_BROXTON(dev_priv)) {
2323                bxt_display_core_uninit(dev_priv);
2324                bxt_enable_dc9(dev_priv);
2325        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2326                hsw_enable_pc8(dev_priv);
2327        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2328                ret = vlv_suspend_complete(dev_priv);
2329        }
2330
2331        if (ret) {
2332                DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
2333                intel_runtime_pm_enable_interrupts(dev_priv);
2334
2335                enable_rpm_wakeref_asserts(dev_priv);
2336
2337                return ret;
2338        }
2339
2340        intel_uncore_forcewake_reset(dev_priv, false);
2341
2342        enable_rpm_wakeref_asserts(dev_priv);
2343        WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2344
2345        if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv))
2346                DRM_ERROR("Unclaimed access detected prior to suspending\n");
2347
2348        dev_priv->pm.suspended = true;
2349
2350        /*
2351         * FIXME: We really should find a document that references the arguments
2352         * used below!
2353         */
2354        if (IS_BROADWELL(dev_priv)) {
2355                /*
2356                 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
2357                 * being detected, and the call we do at intel_runtime_resume()
2358                 * won't be able to restore them. Since PCI_D3hot matches the
2359                 * actual specification and appears to be working, use it.
2360                 */
2361                intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
2362        } else {
2363                /*
2364                 * current versions of firmware which depend on this opregion
2365                 * notification have repurposed the D1 definition to mean
2366                 * "runtime suspended" vs. what you would normally expect (D3)
2367                 * to distinguish it from notifications that might be sent via
2368                 * the suspend path.
2369                 */
2370                intel_opregion_notify_adapter(dev_priv, PCI_D1);
2371        }
2372
2373        assert_forcewakes_inactive(dev_priv);
2374
2375        if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
2376                intel_hpd_poll_init(dev_priv);
2377
2378        DRM_DEBUG_KMS("Device suspended\n");
2379        return 0;
2380}
2381
2382static int intel_runtime_resume(struct device *kdev)
2383{
2384        struct pci_dev *pdev = to_pci_dev(kdev);
2385        struct drm_device *dev = pci_get_drvdata(pdev);
2386        struct drm_i915_private *dev_priv = to_i915(dev);
2387        int ret = 0;
2388
2389        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
2390                return -ENODEV;
2391
2392        DRM_DEBUG_KMS("Resuming device\n");
2393
2394        WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
2395        disable_rpm_wakeref_asserts(dev_priv);
2396
2397        intel_opregion_notify_adapter(dev_priv, PCI_D0);
2398        dev_priv->pm.suspended = false;
2399        if (intel_uncore_unclaimed_mmio(dev_priv))
2400                DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
2401
2402        intel_guc_resume(dev);
2403
2404        if (IS_GEN6(dev_priv))
2405                intel_init_pch_refclk(dev);
2406
2407        if (IS_BROXTON(dev)) {
2408                bxt_disable_dc9(dev_priv);
2409                bxt_display_core_init(dev_priv, true);
2410                if (dev_priv->csr.dmc_payload &&
2411                    (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2412                        gen9_enable_dc5(dev_priv);
2413        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2414                hsw_disable_pc8(dev_priv);
2415        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
2416                ret = vlv_resume_prepare(dev_priv, true);
2417        }
2418
2419        /*
2420         * No point of rolling back things in case of an error, as the best
2421         * we can do is to hope that things will still work (and disable RPM).
2422         */
2423        i915_gem_init_swizzling(dev);
2424
2425        intel_runtime_pm_enable_interrupts(dev_priv);
2426
2427        /*
2428         * On VLV/CHV display interrupts are part of the display
2429         * power well, so hpd is reinitialized from there. For
2430         * everyone else do it here.
2431         */
2432        if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
2433                intel_hpd_init(dev_priv);
2434
2435        enable_rpm_wakeref_asserts(dev_priv);
2436
2437        if (ret)
2438                DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
2439        else
2440                DRM_DEBUG_KMS("Device resumed\n");
2441
2442        return ret;
2443}
2444
2445const struct dev_pm_ops i915_pm_ops = {
2446        /*
2447         * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
2448         * PMSG_RESUME]
2449         */
2450        .suspend = i915_pm_suspend,
2451        .suspend_late = i915_pm_suspend_late,
2452        .resume_early = i915_pm_resume_early,
2453        .resume = i915_pm_resume,
2454
2455        /*
2456         * S4 event handlers
2457         * @freeze, @freeze_late    : called (1) before creating the
2458         *                            hibernation image [PMSG_FREEZE] and
2459         *                            (2) after rebooting, before restoring
2460         *                            the image [PMSG_QUIESCE]
2461         * @thaw, @thaw_early       : called (1) after creating the hibernation
2462         *                            image, before writing it [PMSG_THAW]
2463         *                            and (2) after failing to create or
2464         *                            restore the image [PMSG_RECOVER]
2465         * @poweroff, @poweroff_late: called after writing the hibernation
2466         *                            image, before rebooting [PMSG_HIBERNATE]
2467         * @restore, @restore_early : called after rebooting and restoring the
2468         *                            hibernation image [PMSG_RESTORE]
2469         */
2470        .freeze = i915_pm_freeze,
2471        .freeze_late = i915_pm_freeze_late,
2472        .thaw_early = i915_pm_thaw_early,
2473        .thaw = i915_pm_thaw,
2474        .poweroff = i915_pm_suspend,
2475        .poweroff_late = i915_pm_poweroff_late,
2476        .restore_early = i915_pm_restore_early,
2477        .restore = i915_pm_restore,
2478
2479        /* S0ix (via runtime suspend) event handlers */
2480        .runtime_suspend = intel_runtime_suspend,
2481        .runtime_resume = intel_runtime_resume,
2482};
2483
2484static const struct vm_operations_struct i915_gem_vm_ops = {
2485        .fault = i915_gem_fault,
2486        .open = drm_gem_vm_open,
2487        .close = drm_gem_vm_close,
2488};
2489
2490static const struct file_operations i915_driver_fops = {
2491        .owner = THIS_MODULE,
2492        .open = drm_open,
2493        .release = drm_release,
2494        .unlocked_ioctl = drm_ioctl,
2495        .mmap = drm_gem_mmap,
2496        .poll = drm_poll,
2497        .read = drm_read,
2498#ifdef CONFIG_COMPAT
2499        .compat_ioctl = i915_compat_ioctl,
2500#endif
2501        .llseek = noop_llseek,
2502};
2503
2504static int
2505i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
2506                          struct drm_file *file)
2507{
2508        return -ENODEV;
2509}
2510
2511static const struct drm_ioctl_desc i915_ioctls[] = {
2512        DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2513        DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
2514        DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
2515        DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
2516        DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
2517        DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
2518        DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
2519        DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2520        DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2521        DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2522        DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2523        DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
2524        DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2525        DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2526        DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  drm_noop, DRM_AUTH),
2527        DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
2528        DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2529        DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2530        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
2531        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
2532        DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2533        DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
2534        DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2535        DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
2536        DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
2537        DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2538        DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2539        DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2540        DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
2541        DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
2542        DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
2543        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
2544        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
2545        DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
2546        DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
2547        DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
2548        DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
2549        DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
2550        DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
2551        DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
2552        DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2553        DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
2554        DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
2555        DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
2556        DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
2557        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
2558        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
2559        DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
2560        DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
2561        DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
2562        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
2563        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
2564};
2565
2566static struct drm_driver driver = {
2567        /* Don't use MTRRs here; the Xserver or userspace app should
2568         * deal with them for Intel hardware.
2569         */
2570        .driver_features =
2571            DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
2572            DRIVER_RENDER | DRIVER_MODESET,
2573        .open = i915_driver_open,
2574        .lastclose = i915_driver_lastclose,
2575        .preclose = i915_driver_preclose,
2576        .postclose = i915_driver_postclose,
2577        .set_busid = drm_pci_set_busid,
2578
2579        .gem_close_object = i915_gem_close_object,
2580        .gem_free_object = i915_gem_free_object,
2581        .gem_vm_ops = &i915_gem_vm_ops,
2582
2583        .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
2584        .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
2585        .gem_prime_export = i915_gem_prime_export,
2586        .gem_prime_import = i915_gem_prime_import,
2587
2588        .dumb_create = i915_gem_dumb_create,
2589        .dumb_map_offset = i915_gem_mmap_gtt,
2590        .dumb_destroy = drm_gem_dumb_destroy,
2591        .ioctls = i915_ioctls,
2592        .num_ioctls = ARRAY_SIZE(i915_ioctls),
2593        .fops = &i915_driver_fops,
2594        .name = DRIVER_NAME,
2595        .desc = DRIVER_DESC,
2596        .date = DRIVER_DATE,
2597        .major = DRIVER_MAJOR,
2598        .minor = DRIVER_MINOR,
2599        .patchlevel = DRIVER_PATCHLEVEL,
2600};
Note: See TracBrowser for help on using the repository browser.