source: src/linux/universal/linux-4.9/drivers/gpu/drm/i915/i915_gem.c @ 31574

Last change on this file since 31574 was 31574, checked in by brainslayer, 5 months ago

kernel 4.9 update

File size: 126.7 KB
Line 
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 *    Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include <drm/drmP.h>
29#include <drm/drm_vma_manager.h>
30#include <drm/i915_drm.h>
31#include "i915_drv.h"
32#include "i915_gem_dmabuf.h"
33#include "i915_vgpu.h"
34#include "i915_trace.h"
35#include "intel_drv.h"
36#include "intel_frontbuffer.h"
37#include "intel_mocs.h"
38#include <linux/reservation.h>
39#include <linux/shmem_fs.h>
40#include <linux/slab.h>
41#include <linux/swap.h>
42#include <linux/pci.h>
43#include <linux/dma-buf.h>
44
45static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
46static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
47
48static bool cpu_cache_is_coherent(struct drm_device *dev,
49                                  enum i915_cache_level level)
50{
51        return HAS_LLC(dev) || level != I915_CACHE_NONE;
52}
53
54static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
55{
56        if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
57                return false;
58
59        if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
60                return true;
61
62        return obj->pin_display;
63}
64
65static int
66insert_mappable_node(struct drm_i915_private *i915,
67                     struct drm_mm_node *node, u32 size)
68{
69        memset(node, 0, sizeof(*node));
70        return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
71                                                   size, 0, 0, 0,
72                                                   i915->ggtt.mappable_end,
73                                                   DRM_MM_SEARCH_DEFAULT,
74                                                   DRM_MM_CREATE_DEFAULT);
75}
76
77static void
78remove_mappable_node(struct drm_mm_node *node)
79{
80        drm_mm_remove_node(node);
81}
82
83/* some bookkeeping */
84static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
85                                  size_t size)
86{
87        spin_lock(&dev_priv->mm.object_stat_lock);
88        dev_priv->mm.object_count++;
89        dev_priv->mm.object_memory += size;
90        spin_unlock(&dev_priv->mm.object_stat_lock);
91}
92
93static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
94                                     size_t size)
95{
96        spin_lock(&dev_priv->mm.object_stat_lock);
97        dev_priv->mm.object_count--;
98        dev_priv->mm.object_memory -= size;
99        spin_unlock(&dev_priv->mm.object_stat_lock);
100}
101
102static int
103i915_gem_wait_for_error(struct i915_gpu_error *error)
104{
105        int ret;
106
107        if (!i915_reset_in_progress(error))
108                return 0;
109
110        /*
111         * Only wait 10 seconds for the gpu reset to complete to avoid hanging
112         * userspace. If it takes that long something really bad is going on and
113         * we should simply try to bail out and fail as gracefully as possible.
114         */
115        ret = wait_event_interruptible_timeout(error->reset_queue,
116                                               !i915_reset_in_progress(error),
117                                               10*HZ);
118        if (ret == 0) {
119                DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
120                return -EIO;
121        } else if (ret < 0) {
122                return ret;
123        } else {
124                return 0;
125        }
126}
127
128int i915_mutex_lock_interruptible(struct drm_device *dev)
129{
130        struct drm_i915_private *dev_priv = to_i915(dev);
131        int ret;
132
133        ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
134        if (ret)
135                return ret;
136
137        ret = mutex_lock_interruptible(&dev->struct_mutex);
138        if (ret)
139                return ret;
140
141        return 0;
142}
143
144int
145i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
146                            struct drm_file *file)
147{
148        struct drm_i915_private *dev_priv = to_i915(dev);
149        struct i915_ggtt *ggtt = &dev_priv->ggtt;
150        struct drm_i915_gem_get_aperture *args = data;
151        struct i915_vma *vma;
152        size_t pinned;
153
154        pinned = 0;
155        mutex_lock(&dev->struct_mutex);
156        list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
157                if (i915_vma_is_pinned(vma))
158                        pinned += vma->node.size;
159        list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
160                if (i915_vma_is_pinned(vma))
161                        pinned += vma->node.size;
162        mutex_unlock(&dev->struct_mutex);
163
164        args->aper_size = ggtt->base.total;
165        args->aper_available_size = args->aper_size - pinned;
166
167        return 0;
168}
169
170static int
171i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
172{
173        struct address_space *mapping = obj->base.filp->f_mapping;
174        char *vaddr = obj->phys_handle->vaddr;
175        struct sg_table *st;
176        struct scatterlist *sg;
177        int i;
178
179        if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
180                return -EINVAL;
181
182        for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
183                struct page *page;
184                char *src;
185
186                page = shmem_read_mapping_page(mapping, i);
187                if (IS_ERR(page))
188                        return PTR_ERR(page);
189
190                src = kmap_atomic(page);
191                memcpy(vaddr, src, PAGE_SIZE);
192                drm_clflush_virt_range(vaddr, PAGE_SIZE);
193                kunmap_atomic(src);
194
195                put_page(page);
196                vaddr += PAGE_SIZE;
197        }
198
199        i915_gem_chipset_flush(to_i915(obj->base.dev));
200
201        st = kmalloc(sizeof(*st), GFP_KERNEL);
202        if (st == NULL)
203                return -ENOMEM;
204
205        if (sg_alloc_table(st, 1, GFP_KERNEL)) {
206                kfree(st);
207                return -ENOMEM;
208        }
209
210        sg = st->sgl;
211        sg->offset = 0;
212        sg->length = obj->base.size;
213
214        sg_dma_address(sg) = obj->phys_handle->busaddr;
215        sg_dma_len(sg) = obj->base.size;
216
217        obj->pages = st;
218        return 0;
219}
220
221static void
222i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
223{
224        int ret;
225
226        BUG_ON(obj->madv == __I915_MADV_PURGED);
227
228        ret = i915_gem_object_set_to_cpu_domain(obj, true);
229        if (WARN_ON(ret)) {
230                /* In the event of a disaster, abandon all caches and
231                 * hope for the best.
232                 */
233                obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
234        }
235
236        if (obj->madv == I915_MADV_DONTNEED)
237                obj->dirty = 0;
238
239        if (obj->dirty) {
240                struct address_space *mapping = obj->base.filp->f_mapping;
241                char *vaddr = obj->phys_handle->vaddr;
242                int i;
243
244                for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
245                        struct page *page;
246                        char *dst;
247
248                        page = shmem_read_mapping_page(mapping, i);
249                        if (IS_ERR(page))
250                                continue;
251
252                        dst = kmap_atomic(page);
253                        drm_clflush_virt_range(vaddr, PAGE_SIZE);
254                        memcpy(dst, vaddr, PAGE_SIZE);
255                        kunmap_atomic(dst);
256
257                        set_page_dirty(page);
258                        if (obj->madv == I915_MADV_WILLNEED)
259                                mark_page_accessed(page);
260                        put_page(page);
261                        vaddr += PAGE_SIZE;
262                }
263                obj->dirty = 0;
264        }
265
266        sg_free_table(obj->pages);
267        kfree(obj->pages);
268}
269
270static void
271i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
272{
273        drm_pci_free(obj->base.dev, obj->phys_handle);
274}
275
276static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
277        .get_pages = i915_gem_object_get_pages_phys,
278        .put_pages = i915_gem_object_put_pages_phys,
279        .release = i915_gem_object_release_phys,
280};
281
282int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
283{
284        struct i915_vma *vma;
285        LIST_HEAD(still_in_list);
286        int ret;
287
288        lockdep_assert_held(&obj->base.dev->struct_mutex);
289
290        /* Closed vma are removed from the obj->vma_list - but they may
291         * still have an active binding on the object. To remove those we
292         * must wait for all rendering to complete to the object (as unbinding
293         * must anyway), and retire the requests.
294         */
295        ret = i915_gem_object_wait_rendering(obj, false);
296        if (ret)
297                return ret;
298
299        i915_gem_retire_requests(to_i915(obj->base.dev));
300
301        while ((vma = list_first_entry_or_null(&obj->vma_list,
302                                               struct i915_vma,
303                                               obj_link))) {
304                list_move_tail(&vma->obj_link, &still_in_list);
305                ret = i915_vma_unbind(vma);
306                if (ret)
307                        break;
308        }
309        list_splice(&still_in_list, &obj->vma_list);
310
311        return ret;
312}
313
314/**
315 * Ensures that all rendering to the object has completed and the object is
316 * safe to unbind from the GTT or access from the CPU.
317 * @obj: i915 gem object
318 * @readonly: waiting for just read access or read-write access
319 */
320int
321i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
322                               bool readonly)
323{
324        struct reservation_object *resv;
325        struct i915_gem_active *active;
326        unsigned long active_mask;
327        int idx;
328
329        lockdep_assert_held(&obj->base.dev->struct_mutex);
330
331        if (!readonly) {
332                active = obj->last_read;
333                active_mask = i915_gem_object_get_active(obj);
334        } else {
335                active_mask = 1;
336                active = &obj->last_write;
337        }
338
339        for_each_active(active_mask, idx) {
340                int ret;
341
342                ret = i915_gem_active_wait(&active[idx],
343                                           &obj->base.dev->struct_mutex);
344                if (ret)
345                        return ret;
346        }
347
348        resv = i915_gem_object_get_dmabuf_resv(obj);
349        if (resv) {
350                long err;
351
352                err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
353                                                          MAX_SCHEDULE_TIMEOUT);
354                if (err < 0)
355                        return err;
356        }
357
358        return 0;
359}
360
361/* A nonblocking variant of the above wait. Must be called prior to
362 * acquiring the mutex for the object, as the object state may change
363 * during this call. A reference must be held by the caller for the object.
364 */
365static __must_check int
366__unsafe_wait_rendering(struct drm_i915_gem_object *obj,
367                        struct intel_rps_client *rps,
368                        bool readonly)
369{
370        struct i915_gem_active *active;
371        unsigned long active_mask;
372        int idx;
373
374        active_mask = __I915_BO_ACTIVE(obj);
375        if (!active_mask)
376                return 0;
377
378        if (!readonly) {
379                active = obj->last_read;
380        } else {
381                active_mask = 1;
382                active = &obj->last_write;
383        }
384
385        for_each_active(active_mask, idx) {
386                int ret;
387
388                ret = i915_gem_active_wait_unlocked(&active[idx],
389                                                    I915_WAIT_INTERRUPTIBLE,
390                                                    NULL, rps);
391                if (ret)
392                        return ret;
393        }
394
395        return 0;
396}
397
398static struct intel_rps_client *to_rps_client(struct drm_file *file)
399{
400        struct drm_i915_file_private *fpriv = file->driver_priv;
401
402        return &fpriv->rps;
403}
404
405int
406i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
407                            int align)
408{
409        drm_dma_handle_t *phys;
410        int ret;
411
412        if (obj->phys_handle) {
413                if ((unsigned long)obj->phys_handle->vaddr & (align -1))
414                        return -EBUSY;
415
416                return 0;
417        }
418
419        if (obj->madv != I915_MADV_WILLNEED)
420                return -EFAULT;
421
422        if (obj->base.filp == NULL)
423                return -EINVAL;
424
425        ret = i915_gem_object_unbind(obj);
426        if (ret)
427                return ret;
428
429        ret = i915_gem_object_put_pages(obj);
430        if (ret)
431                return ret;
432
433        /* create a new object */
434        phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
435        if (!phys)
436                return -ENOMEM;
437
438        obj->phys_handle = phys;
439        obj->ops = &i915_gem_phys_ops;
440
441        return i915_gem_object_get_pages(obj);
442}
443
444static int
445i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
446                     struct drm_i915_gem_pwrite *args,
447                     struct drm_file *file_priv)
448{
449        struct drm_device *dev = obj->base.dev;
450        void *vaddr = obj->phys_handle->vaddr + args->offset;
451        char __user *user_data = u64_to_user_ptr(args->data_ptr);
452        int ret = 0;
453
454        /* We manually control the domain here and pretend that it
455         * remains coherent i.e. in the GTT domain, like shmem_pwrite.
456         */
457        ret = i915_gem_object_wait_rendering(obj, false);
458        if (ret)
459                return ret;
460
461        intel_fb_obj_invalidate(obj, ORIGIN_CPU);
462        if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
463                unsigned long unwritten;
464
465                /* The physical object once assigned is fixed for the lifetime
466                 * of the obj, so we can safely drop the lock and continue
467                 * to access vaddr.
468                 */
469                mutex_unlock(&dev->struct_mutex);
470                unwritten = copy_from_user(vaddr, user_data, args->size);
471                mutex_lock(&dev->struct_mutex);
472                if (unwritten) {
473                        ret = -EFAULT;
474                        goto out;
475                }
476        }
477
478        drm_clflush_virt_range(vaddr, args->size);
479        i915_gem_chipset_flush(to_i915(dev));
480
481out:
482        intel_fb_obj_flush(obj, false, ORIGIN_CPU);
483        return ret;
484}
485
486void *i915_gem_object_alloc(struct drm_device *dev)
487{
488        struct drm_i915_private *dev_priv = to_i915(dev);
489        return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
490}
491
492void i915_gem_object_free(struct drm_i915_gem_object *obj)
493{
494        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
495        kmem_cache_free(dev_priv->objects, obj);
496}
497
498static int
499i915_gem_create(struct drm_file *file,
500                struct drm_device *dev,
501                uint64_t size,
502                uint32_t *handle_p)
503{
504        struct drm_i915_gem_object *obj;
505        int ret;
506        u32 handle;
507
508        size = roundup(size, PAGE_SIZE);
509        if (size == 0)
510                return -EINVAL;
511
512        /* Allocate the new object */
513        obj = i915_gem_object_create(dev, size);
514        if (IS_ERR(obj))
515                return PTR_ERR(obj);
516
517        ret = drm_gem_handle_create(file, &obj->base, &handle);
518        /* drop reference from allocate - handle holds it now */
519        i915_gem_object_put_unlocked(obj);
520        if (ret)
521                return ret;
522
523        *handle_p = handle;
524        return 0;
525}
526
527int
528i915_gem_dumb_create(struct drm_file *file,
529                     struct drm_device *dev,
530                     struct drm_mode_create_dumb *args)
531{
532        /* have to work out size/pitch and return them */
533        args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
534        args->size = args->pitch * args->height;
535        return i915_gem_create(file, dev,
536                               args->size, &args->handle);
537}
538
539/**
540 * Creates a new mm object and returns a handle to it.
541 * @dev: drm device pointer
542 * @data: ioctl data blob
543 * @file: drm file pointer
544 */
545int
546i915_gem_create_ioctl(struct drm_device *dev, void *data,
547                      struct drm_file *file)
548{
549        struct drm_i915_gem_create *args = data;
550
551        return i915_gem_create(file, dev,
552                               args->size, &args->handle);
553}
554
555static inline int
556__copy_to_user_swizzled(char __user *cpu_vaddr,
557                        const char *gpu_vaddr, int gpu_offset,
558                        int length)
559{
560        int ret, cpu_offset = 0;
561
562        while (length > 0) {
563                int cacheline_end = ALIGN(gpu_offset + 1, 64);
564                int this_length = min(cacheline_end - gpu_offset, length);
565                int swizzled_gpu_offset = gpu_offset ^ 64;
566
567                ret = __copy_to_user(cpu_vaddr + cpu_offset,
568                                     gpu_vaddr + swizzled_gpu_offset,
569                                     this_length);
570                if (ret)
571                        return ret + length;
572
573                cpu_offset += this_length;
574                gpu_offset += this_length;
575                length -= this_length;
576        }
577
578        return 0;
579}
580
581static inline int
582__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
583                          const char __user *cpu_vaddr,
584                          int length)
585{
586        int ret, cpu_offset = 0;
587
588        while (length > 0) {
589                int cacheline_end = ALIGN(gpu_offset + 1, 64);
590                int this_length = min(cacheline_end - gpu_offset, length);
591                int swizzled_gpu_offset = gpu_offset ^ 64;
592
593                ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
594                                       cpu_vaddr + cpu_offset,
595                                       this_length);
596                if (ret)
597                        return ret + length;
598
599                cpu_offset += this_length;
600                gpu_offset += this_length;
601                length -= this_length;
602        }
603
604        return 0;
605}
606
607/*
608 * Pins the specified object's pages and synchronizes the object with
609 * GPU accesses. Sets needs_clflush to non-zero if the caller should
610 * flush the object from the CPU cache.
611 */
612int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
613                                    unsigned int *needs_clflush)
614{
615        int ret;
616
617        *needs_clflush = 0;
618
619        if (!i915_gem_object_has_struct_page(obj))
620                return -ENODEV;
621
622        ret = i915_gem_object_wait_rendering(obj, true);
623        if (ret)
624                return ret;
625
626        ret = i915_gem_object_get_pages(obj);
627        if (ret)
628                return ret;
629
630        i915_gem_object_pin_pages(obj);
631
632        i915_gem_object_flush_gtt_write_domain(obj);
633
634        /* If we're not in the cpu read domain, set ourself into the gtt
635         * read domain and manually flush cachelines (if required). This
636         * optimizes for the case when the gpu will dirty the data
637         * anyway again before the next pread happens.
638         */
639        if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
640                *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
641                                                        obj->cache_level);
642
643        if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
644                ret = i915_gem_object_set_to_cpu_domain(obj, false);
645                if (ret)
646                        goto err_unpin;
647
648                *needs_clflush = 0;
649        }
650
651        /* return with the pages pinned */
652        return 0;
653
654err_unpin:
655        i915_gem_object_unpin_pages(obj);
656        return ret;
657}
658
659int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
660                                     unsigned int *needs_clflush)
661{
662        int ret;
663
664        *needs_clflush = 0;
665        if (!i915_gem_object_has_struct_page(obj))
666                return -ENODEV;
667
668        ret = i915_gem_object_wait_rendering(obj, false);
669        if (ret)
670                return ret;
671
672        ret = i915_gem_object_get_pages(obj);
673        if (ret)
674                return ret;
675
676        i915_gem_object_pin_pages(obj);
677
678        i915_gem_object_flush_gtt_write_domain(obj);
679
680        /* If we're not in the cpu write domain, set ourself into the
681         * gtt write domain and manually flush cachelines (as required).
682         * This optimizes for the case when the gpu will use the data
683         * right away and we therefore have to clflush anyway.
684         */
685        if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
686                *needs_clflush |= cpu_write_needs_clflush(obj) << 1;
687
688        /* Same trick applies to invalidate partially written cachelines read
689         * before writing.
690         */
691        if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
692                *needs_clflush |= !cpu_cache_is_coherent(obj->base.dev,
693                                                         obj->cache_level);
694
695        if (*needs_clflush && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
696                ret = i915_gem_object_set_to_cpu_domain(obj, true);
697                if (ret)
698                        goto err_unpin;
699
700                *needs_clflush = 0;
701        }
702
703        if ((*needs_clflush & CLFLUSH_AFTER) == 0)
704                obj->cache_dirty = true;
705
706        intel_fb_obj_invalidate(obj, ORIGIN_CPU);
707        obj->dirty = 1;
708        /* return with the pages pinned */
709        return 0;
710
711err_unpin:
712        i915_gem_object_unpin_pages(obj);
713        return ret;
714}
715
716/* Per-page copy function for the shmem pread fastpath.
717 * Flushes invalid cachelines before reading the target if
718 * needs_clflush is set. */
719static int
720shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
721                 char __user *user_data,
722                 bool page_do_bit17_swizzling, bool needs_clflush)
723{
724        char *vaddr;
725        int ret;
726
727        if (unlikely(page_do_bit17_swizzling))
728                return -EINVAL;
729
730        vaddr = kmap_atomic(page);
731        if (needs_clflush)
732                drm_clflush_virt_range(vaddr + shmem_page_offset,
733                                       page_length);
734        ret = __copy_to_user_inatomic(user_data,
735                                      vaddr + shmem_page_offset,
736                                      page_length);
737        kunmap_atomic(vaddr);
738
739        return ret ? -EFAULT : 0;
740}
741
742static void
743shmem_clflush_swizzled_range(char *addr, unsigned long length,
744                             bool swizzled)
745{
746        if (unlikely(swizzled)) {
747                unsigned long start = (unsigned long) addr;
748                unsigned long end = (unsigned long) addr + length;
749
750                /* For swizzling simply ensure that we always flush both
751                 * channels. Lame, but simple and it works. Swizzled
752                 * pwrite/pread is far from a hotpath - current userspace
753                 * doesn't use it at all. */
754                start = round_down(start, 128);
755                end = round_up(end, 128);
756
757                drm_clflush_virt_range((void *)start, end - start);
758        } else {
759                drm_clflush_virt_range(addr, length);
760        }
761
762}
763
764/* Only difference to the fast-path function is that this can handle bit17
765 * and uses non-atomic copy and kmap functions. */
766static int
767shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
768                 char __user *user_data,
769                 bool page_do_bit17_swizzling, bool needs_clflush)
770{
771        char *vaddr;
772        int ret;
773
774        vaddr = kmap(page);
775        if (needs_clflush)
776                shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
777                                             page_length,
778                                             page_do_bit17_swizzling);
779
780        if (page_do_bit17_swizzling)
781                ret = __copy_to_user_swizzled(user_data,
782                                              vaddr, shmem_page_offset,
783                                              page_length);
784        else
785                ret = __copy_to_user(user_data,
786                                     vaddr + shmem_page_offset,
787                                     page_length);
788        kunmap(page);
789
790        return ret ? - EFAULT : 0;
791}
792
793static inline unsigned long
794slow_user_access(struct io_mapping *mapping,
795                 uint64_t page_base, int page_offset,
796                 char __user *user_data,
797                 unsigned long length, bool pwrite)
798{
799        void __iomem *ioaddr;
800        void *vaddr;
801        uint64_t unwritten;
802
803        ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
804        /* We can use the cpu mem copy function because this is X86. */
805        vaddr = (void __force *)ioaddr + page_offset;
806        if (pwrite)
807                unwritten = __copy_from_user(vaddr, user_data, length);
808        else
809                unwritten = __copy_to_user(user_data, vaddr, length);
810
811        io_mapping_unmap(ioaddr);
812        return unwritten;
813}
814
815static int
816i915_gem_gtt_pread(struct drm_device *dev,
817                   struct drm_i915_gem_object *obj, uint64_t size,
818                   uint64_t data_offset, uint64_t data_ptr)
819{
820        struct drm_i915_private *dev_priv = to_i915(dev);
821        struct i915_ggtt *ggtt = &dev_priv->ggtt;
822        struct i915_vma *vma;
823        struct drm_mm_node node;
824        char __user *user_data;
825        uint64_t remain;
826        uint64_t offset;
827        int ret;
828
829        vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
830        if (!IS_ERR(vma)) {
831                node.start = i915_ggtt_offset(vma);
832                node.allocated = false;
833                ret = i915_vma_put_fence(vma);
834                if (ret) {
835                        i915_vma_unpin(vma);
836                        vma = ERR_PTR(ret);
837                }
838        }
839        if (IS_ERR(vma)) {
840                ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
841                if (ret)
842                        goto out;
843
844                ret = i915_gem_object_get_pages(obj);
845                if (ret) {
846                        remove_mappable_node(&node);
847                        goto out;
848                }
849
850                i915_gem_object_pin_pages(obj);
851        }
852
853        ret = i915_gem_object_set_to_gtt_domain(obj, false);
854        if (ret)
855                goto out_unpin;
856
857        user_data = u64_to_user_ptr(data_ptr);
858        remain = size;
859        offset = data_offset;
860
861        mutex_unlock(&dev->struct_mutex);
862        if (likely(!i915.prefault_disable)) {
863                ret = fault_in_pages_writeable(user_data, remain);
864                if (ret) {
865                        mutex_lock(&dev->struct_mutex);
866                        goto out_unpin;
867                }
868        }
869
870        while (remain > 0) {
871                /* Operation in this page
872                 *
873                 * page_base = page offset within aperture
874                 * page_offset = offset within page
875                 * page_length = bytes to copy for this page
876                 */
877                u32 page_base = node.start;
878                unsigned page_offset = offset_in_page(offset);
879                unsigned page_length = PAGE_SIZE - page_offset;
880                page_length = remain < page_length ? remain : page_length;
881                if (node.allocated) {
882                        wmb();
883                        ggtt->base.insert_page(&ggtt->base,
884                                               i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
885                                               node.start,
886                                               I915_CACHE_NONE, 0);
887                        wmb();
888                } else {
889                        page_base += offset & PAGE_MASK;
890                }
891                /* This is a slow read/write as it tries to read from
892                 * and write to user memory which may result into page
893                 * faults, and so we cannot perform this under struct_mutex.
894                 */
895                if (slow_user_access(&ggtt->mappable, page_base,
896                                     page_offset, user_data,
897                                     page_length, false)) {
898                        ret = -EFAULT;
899                        break;
900                }
901
902                remain -= page_length;
903                user_data += page_length;
904                offset += page_length;
905        }
906
907        mutex_lock(&dev->struct_mutex);
908        if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
909                /* The user has modified the object whilst we tried
910                 * reading from it, and we now have no idea what domain
911                 * the pages should be in. As we have just been touching
912                 * them directly, flush everything back to the GTT
913                 * domain.
914                 */
915                ret = i915_gem_object_set_to_gtt_domain(obj, false);
916        }
917
918out_unpin:
919        if (node.allocated) {
920                wmb();
921                ggtt->base.clear_range(&ggtt->base,
922                                       node.start, node.size,
923                                       true);
924                i915_gem_object_unpin_pages(obj);
925                remove_mappable_node(&node);
926        } else {
927                i915_vma_unpin(vma);
928        }
929out:
930        return ret;
931}
932
933static int
934i915_gem_shmem_pread(struct drm_device *dev,
935                     struct drm_i915_gem_object *obj,
936                     struct drm_i915_gem_pread *args,
937                     struct drm_file *file)
938{
939        char __user *user_data;
940        ssize_t remain;
941        loff_t offset;
942        int shmem_page_offset, page_length, ret = 0;
943        int obj_do_bit17_swizzling, page_do_bit17_swizzling;
944        int prefaulted = 0;
945        int needs_clflush = 0;
946        struct sg_page_iter sg_iter;
947
948        ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
949        if (ret)
950                return ret;
951
952        obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
953        user_data = u64_to_user_ptr(args->data_ptr);
954        offset = args->offset;
955        remain = args->size;
956
957        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
958                         offset >> PAGE_SHIFT) {
959                struct page *page = sg_page_iter_page(&sg_iter);
960
961                if (remain <= 0)
962                        break;
963
964                /* Operation in this page
965                 *
966                 * shmem_page_offset = offset within page in shmem file
967                 * page_length = bytes to copy for this page
968                 */
969                shmem_page_offset = offset_in_page(offset);
970                page_length = remain;
971                if ((shmem_page_offset + page_length) > PAGE_SIZE)
972                        page_length = PAGE_SIZE - shmem_page_offset;
973
974                page_do_bit17_swizzling = obj_do_bit17_swizzling &&
975                        (page_to_phys(page) & (1 << 17)) != 0;
976
977                ret = shmem_pread_fast(page, shmem_page_offset, page_length,
978                                       user_data, page_do_bit17_swizzling,
979                                       needs_clflush);
980                if (ret == 0)
981                        goto next_page;
982
983                mutex_unlock(&dev->struct_mutex);
984
985                if (likely(!i915.prefault_disable) && !prefaulted) {
986                        ret = fault_in_pages_writeable(user_data, remain);
987                        /* Userspace is tricking us, but we've already clobbered
988                         * its pages with the prefault and promised to write the
989                         * data up to the first fault. Hence ignore any errors
990                         * and just continue. */
991                        (void)ret;
992                        prefaulted = 1;
993                }
994
995                ret = shmem_pread_slow(page, shmem_page_offset, page_length,
996                                       user_data, page_do_bit17_swizzling,
997                                       needs_clflush);
998
999                mutex_lock(&dev->struct_mutex);
1000
1001                if (ret)
1002                        goto out;
1003
1004next_page:
1005                remain -= page_length;
1006                user_data += page_length;
1007                offset += page_length;
1008        }
1009
1010out:
1011        i915_gem_obj_finish_shmem_access(obj);
1012
1013        return ret;
1014}
1015
1016/**
1017 * Reads data from the object referenced by handle.
1018 * @dev: drm device pointer
1019 * @data: ioctl data blob
1020 * @file: drm file pointer
1021 *
1022 * On error, the contents of *data are undefined.
1023 */
1024int
1025i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1026                     struct drm_file *file)
1027{
1028        struct drm_i915_gem_pread *args = data;
1029        struct drm_i915_gem_object *obj;
1030        int ret = 0;
1031
1032        if (args->size == 0)
1033                return 0;
1034
1035        if (!access_ok(VERIFY_WRITE,
1036                       u64_to_user_ptr(args->data_ptr),
1037                       args->size))
1038                return -EFAULT;
1039
1040        obj = i915_gem_object_lookup(file, args->handle);
1041        if (!obj)
1042                return -ENOENT;
1043
1044        /* Bounds check source.  */
1045        if (args->offset > obj->base.size ||
1046            args->size > obj->base.size - args->offset) {
1047                ret = -EINVAL;
1048                goto err;
1049        }
1050
1051        trace_i915_gem_object_pread(obj, args->offset, args->size);
1052
1053        ret = __unsafe_wait_rendering(obj, to_rps_client(file), true);
1054        if (ret)
1055                goto err;
1056
1057        ret = i915_mutex_lock_interruptible(dev);
1058        if (ret)
1059                goto err;
1060
1061        ret = i915_gem_shmem_pread(dev, obj, args, file);
1062
1063        /* pread for non shmem backed objects */
1064        if (ret == -EFAULT || ret == -ENODEV) {
1065                intel_runtime_pm_get(to_i915(dev));
1066                ret = i915_gem_gtt_pread(dev, obj, args->size,
1067                                        args->offset, args->data_ptr);
1068                intel_runtime_pm_put(to_i915(dev));
1069        }
1070
1071        i915_gem_object_put(obj);
1072        mutex_unlock(&dev->struct_mutex);
1073
1074        return ret;
1075
1076err:
1077        i915_gem_object_put_unlocked(obj);
1078        return ret;
1079}
1080
1081/* This is the fast write path which cannot handle
1082 * page faults in the source data
1083 */
1084
1085static inline int
1086fast_user_write(struct io_mapping *mapping,
1087                loff_t page_base, int page_offset,
1088                char __user *user_data,
1089                int length)
1090{
1091        void __iomem *vaddr_atomic;
1092        void *vaddr;
1093        unsigned long unwritten;
1094
1095        vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
1096        /* We can use the cpu mem copy function because this is X86. */
1097        vaddr = (void __force*)vaddr_atomic + page_offset;
1098        unwritten = __copy_from_user_inatomic_nocache(vaddr,
1099                                                      user_data, length);
1100        io_mapping_unmap_atomic(vaddr_atomic);
1101        return unwritten;
1102}
1103
1104/**
1105 * This is the fast pwrite path, where we copy the data directly from the
1106 * user into the GTT, uncached.
1107 * @i915: i915 device private data
1108 * @obj: i915 gem object
1109 * @args: pwrite arguments structure
1110 * @file: drm file pointer
1111 */
1112static int
1113i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
1114                         struct drm_i915_gem_object *obj,
1115                         struct drm_i915_gem_pwrite *args,
1116                         struct drm_file *file)
1117{
1118        struct i915_ggtt *ggtt = &i915->ggtt;
1119        struct drm_device *dev = obj->base.dev;
1120        struct i915_vma *vma;
1121        struct drm_mm_node node;
1122        uint64_t remain, offset;
1123        char __user *user_data;
1124        int ret;
1125        bool hit_slow_path = false;
1126
1127        if (i915_gem_object_is_tiled(obj))
1128                return -EFAULT;
1129
1130        vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1131                                       PIN_MAPPABLE | PIN_NONBLOCK);
1132        if (!IS_ERR(vma)) {
1133                node.start = i915_ggtt_offset(vma);
1134                node.allocated = false;
1135                ret = i915_vma_put_fence(vma);
1136                if (ret) {
1137                        i915_vma_unpin(vma);
1138                        vma = ERR_PTR(ret);
1139                }
1140        }
1141        if (IS_ERR(vma)) {
1142                ret = insert_mappable_node(i915, &node, PAGE_SIZE);
1143                if (ret)
1144                        goto out;
1145
1146                ret = i915_gem_object_get_pages(obj);
1147                if (ret) {
1148                        remove_mappable_node(&node);
1149                        goto out;
1150                }
1151
1152                i915_gem_object_pin_pages(obj);
1153        }
1154
1155        ret = i915_gem_object_set_to_gtt_domain(obj, true);
1156        if (ret)
1157                goto out_unpin;
1158
1159        intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1160        obj->dirty = true;
1161
1162        user_data = u64_to_user_ptr(args->data_ptr);
1163        offset = args->offset;
1164        remain = args->size;
1165        while (remain) {
1166                /* Operation in this page
1167                 *
1168                 * page_base = page offset within aperture
1169                 * page_offset = offset within page
1170                 * page_length = bytes to copy for this page
1171                 */
1172                u32 page_base = node.start;
1173                unsigned page_offset = offset_in_page(offset);
1174                unsigned page_length = PAGE_SIZE - page_offset;
1175                page_length = remain < page_length ? remain : page_length;
1176                if (node.allocated) {
1177                        wmb(); /* flush the write before we modify the GGTT */
1178                        ggtt->base.insert_page(&ggtt->base,
1179                                               i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1180                                               node.start, I915_CACHE_NONE, 0);
1181                        wmb(); /* flush modifications to the GGTT (insert_page) */
1182                } else {
1183                        page_base += offset & PAGE_MASK;
1184                }
1185                /* If we get a fault while copying data, then (presumably) our
1186                 * source page isn't available.  Return the error and we'll
1187                 * retry in the slow path.
1188                 * If the object is non-shmem backed, we retry again with the
1189                 * path that handles page fault.
1190                 */
1191                if (fast_user_write(&ggtt->mappable, page_base,
1192                                    page_offset, user_data, page_length)) {
1193                        hit_slow_path = true;
1194                        mutex_unlock(&dev->struct_mutex);
1195                        if (slow_user_access(&ggtt->mappable,
1196                                             page_base,
1197                                             page_offset, user_data,
1198                                             page_length, true)) {
1199                                ret = -EFAULT;
1200                                mutex_lock(&dev->struct_mutex);
1201                                goto out_flush;
1202                        }
1203
1204                        mutex_lock(&dev->struct_mutex);
1205                }
1206
1207                remain -= page_length;
1208                user_data += page_length;
1209                offset += page_length;
1210        }
1211
1212out_flush:
1213        if (hit_slow_path) {
1214                if (ret == 0 &&
1215                    (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1216                        /* The user has modified the object whilst we tried
1217                         * reading from it, and we now have no idea what domain
1218                         * the pages should be in. As we have just been touching
1219                         * them directly, flush everything back to the GTT
1220                         * domain.
1221                         */
1222                        ret = i915_gem_object_set_to_gtt_domain(obj, false);
1223                }
1224        }
1225
1226        intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1227out_unpin:
1228        if (node.allocated) {
1229                wmb();
1230                ggtt->base.clear_range(&ggtt->base,
1231                                       node.start, node.size,
1232                                       true);
1233                i915_gem_object_unpin_pages(obj);
1234                remove_mappable_node(&node);
1235        } else {
1236                i915_vma_unpin(vma);
1237        }
1238out:
1239        return ret;
1240}
1241
1242/* Per-page copy function for the shmem pwrite fastpath.
1243 * Flushes invalid cachelines before writing to the target if
1244 * needs_clflush_before is set and flushes out any written cachelines after
1245 * writing if needs_clflush is set. */
1246static int
1247shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1248                  char __user *user_data,
1249                  bool page_do_bit17_swizzling,
1250                  bool needs_clflush_before,
1251                  bool needs_clflush_after)
1252{
1253        char *vaddr;
1254        int ret;
1255
1256        if (unlikely(page_do_bit17_swizzling))
1257                return -EINVAL;
1258
1259        vaddr = kmap_atomic(page);
1260        if (needs_clflush_before)
1261                drm_clflush_virt_range(vaddr + shmem_page_offset,
1262                                       page_length);
1263        ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1264                                        user_data, page_length);
1265        if (needs_clflush_after)
1266                drm_clflush_virt_range(vaddr + shmem_page_offset,
1267                                       page_length);
1268        kunmap_atomic(vaddr);
1269
1270        return ret ? -EFAULT : 0;
1271}
1272
1273/* Only difference to the fast-path function is that this can handle bit17
1274 * and uses non-atomic copy and kmap functions. */
1275static int
1276shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1277                  char __user *user_data,
1278                  bool page_do_bit17_swizzling,
1279                  bool needs_clflush_before,
1280                  bool needs_clflush_after)
1281{
1282        char *vaddr;
1283        int ret;
1284
1285        vaddr = kmap(page);
1286        if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1287                shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1288                                             page_length,
1289                                             page_do_bit17_swizzling);
1290        if (page_do_bit17_swizzling)
1291                ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1292                                                user_data,
1293                                                page_length);
1294        else
1295                ret = __copy_from_user(vaddr + shmem_page_offset,
1296                                       user_data,
1297                                       page_length);
1298        if (needs_clflush_after)
1299                shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1300                                             page_length,
1301                                             page_do_bit17_swizzling);
1302        kunmap(page);
1303
1304        return ret ? -EFAULT : 0;
1305}
1306
1307static int
1308i915_gem_shmem_pwrite(struct drm_device *dev,
1309                      struct drm_i915_gem_object *obj,
1310                      struct drm_i915_gem_pwrite *args,
1311                      struct drm_file *file)
1312{
1313        ssize_t remain;
1314        loff_t offset;
1315        char __user *user_data;
1316        int shmem_page_offset, page_length, ret = 0;
1317        int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1318        int hit_slowpath = 0;
1319        unsigned int needs_clflush;
1320        struct sg_page_iter sg_iter;
1321
1322        ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1323        if (ret)
1324                return ret;
1325
1326        obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1327        user_data = u64_to_user_ptr(args->data_ptr);
1328        offset = args->offset;
1329        remain = args->size;
1330
1331        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1332                         offset >> PAGE_SHIFT) {
1333                struct page *page = sg_page_iter_page(&sg_iter);
1334                int partial_cacheline_write;
1335
1336                if (remain <= 0)
1337                        break;
1338
1339                /* Operation in this page
1340                 *
1341                 * shmem_page_offset = offset within page in shmem file
1342                 * page_length = bytes to copy for this page
1343                 */
1344                shmem_page_offset = offset_in_page(offset);
1345
1346                page_length = remain;
1347                if ((shmem_page_offset + page_length) > PAGE_SIZE)
1348                        page_length = PAGE_SIZE - shmem_page_offset;
1349
1350                /* If we don't overwrite a cacheline completely we need to be
1351                 * careful to have up-to-date data by first clflushing. Don't
1352                 * overcomplicate things and flush the entire patch. */
1353                partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE &&
1354                        ((shmem_page_offset | page_length)
1355                                & (boot_cpu_data.x86_clflush_size - 1));
1356
1357                page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1358                        (page_to_phys(page) & (1 << 17)) != 0;
1359
1360                ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1361                                        user_data, page_do_bit17_swizzling,
1362                                        partial_cacheline_write,
1363                                        needs_clflush & CLFLUSH_AFTER);
1364                if (ret == 0)
1365                        goto next_page;
1366
1367                hit_slowpath = 1;
1368                mutex_unlock(&dev->struct_mutex);
1369                ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1370                                        user_data, page_do_bit17_swizzling,
1371                                        partial_cacheline_write,
1372                                        needs_clflush & CLFLUSH_AFTER);
1373
1374                mutex_lock(&dev->struct_mutex);
1375
1376                if (ret)
1377                        goto out;
1378
1379next_page:
1380                remain -= page_length;
1381                user_data += page_length;
1382                offset += page_length;
1383        }
1384
1385out:
1386        i915_gem_obj_finish_shmem_access(obj);
1387
1388        if (hit_slowpath) {
1389                /*
1390                 * Fixup: Flush cpu caches in case we didn't flush the dirty
1391                 * cachelines in-line while writing and the object moved
1392                 * out of the cpu write domain while we've dropped the lock.
1393                 */
1394                if (!(needs_clflush & CLFLUSH_AFTER) &&
1395                    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1396                        if (i915_gem_clflush_object(obj, obj->pin_display))
1397                                needs_clflush |= CLFLUSH_AFTER;
1398                }
1399        }
1400
1401        if (needs_clflush & CLFLUSH_AFTER)
1402                i915_gem_chipset_flush(to_i915(dev));
1403
1404        intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1405        return ret;
1406}
1407
1408/**
1409 * Writes data to the object referenced by handle.
1410 * @dev: drm device
1411 * @data: ioctl data blob
1412 * @file: drm file
1413 *
1414 * On error, the contents of the buffer that were to be modified are undefined.
1415 */
1416int
1417i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1418                      struct drm_file *file)
1419{
1420        struct drm_i915_private *dev_priv = to_i915(dev);
1421        struct drm_i915_gem_pwrite *args = data;
1422        struct drm_i915_gem_object *obj;
1423        int ret;
1424
1425        if (args->size == 0)
1426                return 0;
1427
1428        if (!access_ok(VERIFY_READ,
1429                       u64_to_user_ptr(args->data_ptr),
1430                       args->size))
1431                return -EFAULT;
1432
1433        if (likely(!i915.prefault_disable)) {
1434                ret = fault_in_pages_readable(u64_to_user_ptr(args->data_ptr),
1435                                                   args->size);
1436                if (ret)
1437                        return -EFAULT;
1438        }
1439
1440        obj = i915_gem_object_lookup(file, args->handle);
1441        if (!obj)
1442                return -ENOENT;
1443
1444        /* Bounds check destination. */
1445        if (args->offset > obj->base.size ||
1446            args->size > obj->base.size - args->offset) {
1447                ret = -EINVAL;
1448                goto err;
1449        }
1450
1451        trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1452
1453        ret = __unsafe_wait_rendering(obj, to_rps_client(file), false);
1454        if (ret)
1455                goto err;
1456
1457        intel_runtime_pm_get(dev_priv);
1458
1459        ret = i915_mutex_lock_interruptible(dev);
1460        if (ret)
1461                goto err_rpm;
1462
1463        ret = -EFAULT;
1464        /* We can only do the GTT pwrite on untiled buffers, as otherwise
1465         * it would end up going through the fenced access, and we'll get
1466         * different detiling behavior between reading and writing.
1467         * pread/pwrite currently are reading and writing from the CPU
1468         * perspective, requiring manual detiling by the client.
1469         */
1470        if (!i915_gem_object_has_struct_page(obj) ||
1471            cpu_write_needs_clflush(obj)) {
1472                ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1473                /* Note that the gtt paths might fail with non-page-backed user
1474                 * pointers (e.g. gtt mappings when moving data between
1475                 * textures). Fallback to the shmem path in that case. */
1476        }
1477
1478        if (ret == -EFAULT || ret == -ENOSPC) {
1479                if (obj->phys_handle)
1480                        ret = i915_gem_phys_pwrite(obj, args, file);
1481                else
1482                        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1483        }
1484
1485        i915_gem_object_put(obj);
1486        mutex_unlock(&dev->struct_mutex);
1487        intel_runtime_pm_put(dev_priv);
1488
1489        return ret;
1490
1491err_rpm:
1492        intel_runtime_pm_put(dev_priv);
1493err:
1494        i915_gem_object_put_unlocked(obj);
1495        return ret;
1496}
1497
1498static inline enum fb_op_origin
1499write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1500{
1501        return (domain == I915_GEM_DOMAIN_GTT ?
1502                obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
1503}
1504
1505/**
1506 * Called when user space prepares to use an object with the CPU, either
1507 * through the mmap ioctl's mapping or a GTT mapping.
1508 * @dev: drm device
1509 * @data: ioctl data blob
1510 * @file: drm file
1511 */
1512int
1513i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1514                          struct drm_file *file)
1515{
1516        struct drm_i915_gem_set_domain *args = data;
1517        struct drm_i915_gem_object *obj;
1518        uint32_t read_domains = args->read_domains;
1519        uint32_t write_domain = args->write_domain;
1520        int ret;
1521
1522        /* Only handle setting domains to types used by the CPU. */
1523        if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1524                return -EINVAL;
1525
1526        /* Having something in the write domain implies it's in the read
1527         * domain, and only that read domain.  Enforce that in the request.
1528         */
1529        if (write_domain != 0 && read_domains != write_domain)
1530                return -EINVAL;
1531
1532        obj = i915_gem_object_lookup(file, args->handle);
1533        if (!obj)
1534                return -ENOENT;
1535
1536        /* Try to flush the object off the GPU without holding the lock.
1537         * We will repeat the flush holding the lock in the normal manner
1538         * to catch cases where we are gazumped.
1539         */
1540        ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain);
1541        if (ret)
1542                goto err;
1543
1544        ret = i915_mutex_lock_interruptible(dev);
1545        if (ret)
1546                goto err;
1547
1548        if (read_domains & I915_GEM_DOMAIN_GTT)
1549                ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1550        else
1551                ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1552
1553        if (write_domain != 0)
1554                intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1555
1556        i915_gem_object_put(obj);
1557        mutex_unlock(&dev->struct_mutex);
1558        return ret;
1559
1560err:
1561        i915_gem_object_put_unlocked(obj);
1562        return ret;
1563}
1564
1565/**
1566 * Called when user space has done writes to this buffer
1567 * @dev: drm device
1568 * @data: ioctl data blob
1569 * @file: drm file
1570 */
1571int
1572i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1573                         struct drm_file *file)
1574{
1575        struct drm_i915_gem_sw_finish *args = data;
1576        struct drm_i915_gem_object *obj;
1577        int err = 0;
1578
1579        obj = i915_gem_object_lookup(file, args->handle);
1580        if (!obj)
1581                return -ENOENT;
1582
1583        /* Pinned buffers may be scanout, so flush the cache */
1584        if (READ_ONCE(obj->pin_display)) {
1585                err = i915_mutex_lock_interruptible(dev);
1586                if (!err) {
1587                        i915_gem_object_flush_cpu_write_domain(obj);
1588                        mutex_unlock(&dev->struct_mutex);
1589                }
1590        }
1591
1592        i915_gem_object_put_unlocked(obj);
1593        return err;
1594}
1595
1596/**
1597 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1598 *                       it is mapped to.
1599 * @dev: drm device
1600 * @data: ioctl data blob
1601 * @file: drm file
1602 *
1603 * While the mapping holds a reference on the contents of the object, it doesn't
1604 * imply a ref on the object itself.
1605 *
1606 * IMPORTANT:
1607 *
1608 * DRM driver writers who look a this function as an example for how to do GEM
1609 * mmap support, please don't implement mmap support like here. The modern way
1610 * to implement DRM mmap support is with an mmap offset ioctl (like
1611 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1612 * That way debug tooling like valgrind will understand what's going on, hiding
1613 * the mmap call in a driver private ioctl will break that. The i915 driver only
1614 * does cpu mmaps this way because we didn't know better.
1615 */
1616int
1617i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1618                    struct drm_file *file)
1619{
1620        struct drm_i915_gem_mmap *args = data;
1621        struct drm_i915_gem_object *obj;
1622        unsigned long addr;
1623
1624        if (args->flags & ~(I915_MMAP_WC))
1625                return -EINVAL;
1626
1627        if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1628                return -ENODEV;
1629
1630        obj = i915_gem_object_lookup(file, args->handle);
1631        if (!obj)
1632                return -ENOENT;
1633
1634        /* prime objects have no backing filp to GEM mmap
1635         * pages from.
1636         */
1637        if (!obj->base.filp) {
1638                i915_gem_object_put_unlocked(obj);
1639                return -EINVAL;
1640        }
1641
1642        addr = vm_mmap(obj->base.filp, 0, args->size,
1643                       PROT_READ | PROT_WRITE, MAP_SHARED,
1644                       args->offset);
1645        if (args->flags & I915_MMAP_WC) {
1646                struct mm_struct *mm = current->mm;
1647                struct vm_area_struct *vma;
1648
1649                if (down_write_killable(&mm->mmap_sem)) {
1650                        i915_gem_object_put_unlocked(obj);
1651                        return -EINTR;
1652                }
1653                vma = find_vma(mm, addr);
1654                if (vma)
1655                        vma->vm_page_prot =
1656                                pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1657                else
1658                        addr = -ENOMEM;
1659                up_write(&mm->mmap_sem);
1660
1661                /* This may race, but that's ok, it only gets set */
1662                WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1663        }
1664        i915_gem_object_put_unlocked(obj);
1665        if (IS_ERR((void *)addr))
1666                return addr;
1667
1668        args->addr_ptr = (uint64_t) addr;
1669
1670        return 0;
1671}
1672
1673static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
1674{
1675        u64 size;
1676
1677        size = i915_gem_object_get_stride(obj);
1678        size *= i915_gem_object_get_tiling(obj) == I915_TILING_Y ? 32 : 8;
1679
1680        return size >> PAGE_SHIFT;
1681}
1682
1683/**
1684 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
1685 *
1686 * A history of the GTT mmap interface:
1687 *
1688 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
1689 *     aligned and suitable for fencing, and still fit into the available
1690 *     mappable space left by the pinned display objects. A classic problem
1691 *     we called the page-fault-of-doom where we would ping-pong between
1692 *     two objects that could not fit inside the GTT and so the memcpy
1693 *     would page one object in at the expense of the other between every
1694 *     single byte.
1695 *
1696 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
1697 *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
1698 *     object is too large for the available space (or simply too large
1699 *     for the mappable aperture!), a view is created instead and faulted
1700 *     into userspace. (This view is aligned and sized appropriately for
1701 *     fenced access.)
1702 *
1703 * Restrictions:
1704 *
1705 *  * snoopable objects cannot be accessed via the GTT. It can cause machine
1706 *    hangs on some architectures, corruption on others. An attempt to service
1707 *    a GTT page fault from a snoopable object will generate a SIGBUS.
1708 *
1709 *  * the object must be able to fit into RAM (physical memory, though no
1710 *    limited to the mappable aperture).
1711 *
1712 *
1713 * Caveats:
1714 *
1715 *  * a new GTT page fault will synchronize rendering from the GPU and flush
1716 *    all data to system memory. Subsequent access will not be synchronized.
1717 *
1718 *  * all mappings are revoked on runtime device suspend.
1719 *
1720 *  * there are only 8, 16 or 32 fence registers to share between all users
1721 *    (older machines require fence register for display and blitter access
1722 *    as well). Contention of the fence registers will cause the previous users
1723 *    to be unmapped and any new access will generate new page faults.
1724 *
1725 *  * running out of memory while servicing a fault may generate a SIGBUS,
1726 *    rather than the expected SIGSEGV.
1727 */
1728int i915_gem_mmap_gtt_version(void)
1729{
1730        return 1;
1731}
1732
1733/**
1734 * i915_gem_fault - fault a page into the GTT
1735 * @area: CPU VMA in question
1736 * @vmf: fault info
1737 *
1738 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1739 * from userspace.  The fault handler takes care of binding the object to
1740 * the GTT (if needed), allocating and programming a fence register (again,
1741 * only if needed based on whether the old reg is still valid or the object
1742 * is tiled) and inserting a new PTE into the faulting process.
1743 *
1744 * Note that the faulting process may involve evicting existing objects
1745 * from the GTT and/or fence registers to make room.  So performance may
1746 * suffer if the GTT working set is large or there are few fence registers
1747 * left.
1748 *
1749 * The current feature set supported by i915_gem_fault() and thus GTT mmaps
1750 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
1751 */
1752int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
1753{
1754#define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */
1755        struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
1756        struct drm_device *dev = obj->base.dev;
1757        struct drm_i915_private *dev_priv = to_i915(dev);
1758        struct i915_ggtt *ggtt = &dev_priv->ggtt;
1759        bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1760        struct i915_vma *vma;
1761        pgoff_t page_offset;
1762        unsigned int flags;
1763        int ret;
1764
1765        /* We don't use vmf->pgoff since that has the fake offset */
1766        page_offset = ((unsigned long)vmf->virtual_address - area->vm_start) >>
1767                PAGE_SHIFT;
1768
1769        trace_i915_gem_object_fault(obj, page_offset, true, write);
1770
1771        /* Try to flush the object off the GPU first without holding the lock.
1772         * Upon acquiring the lock, we will perform our sanity checks and then
1773         * repeat the flush holding the lock in the normal manner to catch cases
1774         * where we are gazumped.
1775         */
1776        ret = __unsafe_wait_rendering(obj, NULL, !write);
1777        if (ret)
1778                goto err;
1779
1780        intel_runtime_pm_get(dev_priv);
1781
1782        ret = i915_mutex_lock_interruptible(dev);
1783        if (ret)
1784                goto err_rpm;
1785
1786        /* Access to snoopable pages through the GTT is incoherent. */
1787        if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1788                ret = -EFAULT;
1789                goto err_unlock;
1790        }
1791
1792        /* If the object is smaller than a couple of partial vma, it is
1793         * not worth only creating a single partial vma - we may as well
1794         * clear enough space for the full object.
1795         */
1796        flags = PIN_MAPPABLE;
1797        if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
1798                flags |= PIN_NONBLOCK | PIN_NONFAULT;
1799
1800        /* Now pin it into the GTT as needed */
1801        vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
1802        if (IS_ERR(vma)) {
1803                struct i915_ggtt_view view;
1804                unsigned int chunk_size;
1805
1806                /* Use a partial view if it is bigger than available space */
1807                chunk_size = MIN_CHUNK_PAGES;
1808                if (i915_gem_object_is_tiled(obj))
1809                        chunk_size = roundup(chunk_size, tile_row_pages(obj));
1810
1811                memset(&view, 0, sizeof(view));
1812                view.type = I915_GGTT_VIEW_PARTIAL;
1813                view.params.partial.offset = rounddown(page_offset, chunk_size);
1814                view.params.partial.size =
1815                        min_t(unsigned int, chunk_size,
1816                              (area->vm_end - area->vm_start) / PAGE_SIZE -
1817                              view.params.partial.offset);
1818
1819                /* If the partial covers the entire object, just create a
1820                 * normal VMA.
1821                 */
1822                if (chunk_size >= obj->base.size >> PAGE_SHIFT)
1823                        view.type = I915_GGTT_VIEW_NORMAL;
1824
1825                /* Userspace is now writing through an untracked VMA, abandon
1826                 * all hope that the hardware is able to track future writes.
1827                 */
1828                obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
1829
1830                vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1831        }
1832        if (IS_ERR(vma)) {
1833                ret = PTR_ERR(vma);
1834                goto err_unlock;
1835        }
1836
1837        ret = i915_gem_object_set_to_gtt_domain(obj, write);
1838        if (ret)
1839                goto err_unpin;
1840
1841        ret = i915_vma_get_fence(vma);
1842        if (ret)
1843                goto err_unpin;
1844
1845        /* Finally, remap it using the new GTT offset */
1846        ret = remap_io_mapping(area,
1847                               area->vm_start + (vma->ggtt_view.params.partial.offset << PAGE_SHIFT),
1848                               (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
1849                               min_t(u64, vma->size, area->vm_end - area->vm_start),
1850                               &ggtt->mappable);
1851        if (ret)
1852                goto err_unpin;
1853
1854        obj->fault_mappable = true;
1855err_unpin:
1856        __i915_vma_unpin(vma);
1857err_unlock:
1858        mutex_unlock(&dev->struct_mutex);
1859err_rpm:
1860        intel_runtime_pm_put(dev_priv);
1861err:
1862        switch (ret) {
1863        case -EIO:
1864                /*
1865                 * We eat errors when the gpu is terminally wedged to avoid
1866                 * userspace unduly crashing (gl has no provisions for mmaps to
1867                 * fail). But any other -EIO isn't ours (e.g. swap in failure)
1868                 * and so needs to be reported.
1869                 */
1870                if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1871                        ret = VM_FAULT_SIGBUS;
1872                        break;
1873                }
1874        case -EAGAIN:
1875                /*
1876                 * EAGAIN means the gpu is hung and we'll wait for the error
1877                 * handler to reset everything when re-faulting in
1878                 * i915_mutex_lock_interruptible.
1879                 */
1880        case 0:
1881        case -ERESTARTSYS:
1882        case -EINTR:
1883        case -EBUSY:
1884                /*
1885                 * EBUSY is ok: this just means that another thread
1886                 * already did the job.
1887                 */
1888                ret = VM_FAULT_NOPAGE;
1889                break;
1890        case -ENOMEM:
1891                ret = VM_FAULT_OOM;
1892                break;
1893        case -ENOSPC:
1894        case -EFAULT:
1895                ret = VM_FAULT_SIGBUS;
1896                break;
1897        default:
1898                WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1899                ret = VM_FAULT_SIGBUS;
1900                break;
1901        }
1902        return ret;
1903}
1904
1905/**
1906 * i915_gem_release_mmap - remove physical page mappings
1907 * @obj: obj in question
1908 *
1909 * Preserve the reservation of the mmapping with the DRM core code, but
1910 * relinquish ownership of the pages back to the system.
1911 *
1912 * It is vital that we remove the page mapping if we have mapped a tiled
1913 * object through the GTT and then lose the fence register due to
1914 * resource pressure. Similarly if the object has been moved out of the
1915 * aperture, than pages mapped into userspace must be revoked. Removing the
1916 * mapping will then trigger a page fault on the next user access, allowing
1917 * fixup by i915_gem_fault().
1918 */
1919void
1920i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1921{
1922        /* Serialisation between user GTT access and our code depends upon
1923         * revoking the CPU's PTE whilst the mutex is held. The next user
1924         * pagefault then has to wait until we release the mutex.
1925         */
1926        lockdep_assert_held(&obj->base.dev->struct_mutex);
1927
1928        if (!obj->fault_mappable)
1929                return;
1930
1931        drm_vma_node_unmap(&obj->base.vma_node,
1932                           obj->base.dev->anon_inode->i_mapping);
1933
1934        /* Ensure that the CPU's PTE are revoked and there are not outstanding
1935         * memory transactions from userspace before we return. The TLB
1936         * flushing implied above by changing the PTE above *should* be
1937         * sufficient, an extra barrier here just provides us with a bit
1938         * of paranoid documentation about our requirement to serialise
1939         * memory writes before touching registers / GSM.
1940         */
1941        wmb();
1942
1943        obj->fault_mappable = false;
1944}
1945
1946void
1947i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1948{
1949        struct drm_i915_gem_object *obj;
1950
1951        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1952                i915_gem_release_mmap(obj);
1953}
1954
1955/**
1956 * i915_gem_get_ggtt_size - return required global GTT size for an object
1957 * @dev_priv: i915 device
1958 * @size: object size
1959 * @tiling_mode: tiling mode
1960 *
1961 * Return the required global GTT size for an object, taking into account
1962 * potential fence register mapping.
1963 */
1964u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv,
1965                           u64 size, int tiling_mode)
1966{
1967        u64 ggtt_size;
1968
1969        GEM_BUG_ON(size == 0);
1970
1971        if (INTEL_GEN(dev_priv) >= 4 ||
1972            tiling_mode == I915_TILING_NONE)
1973                return size;
1974
1975        /* Previous chips need a power-of-two fence region when tiling */
1976        if (IS_GEN3(dev_priv))
1977                ggtt_size = 1024*1024;
1978        else
1979                ggtt_size = 512*1024;
1980
1981        while (ggtt_size < size)
1982                ggtt_size <<= 1;
1983
1984        return ggtt_size;
1985}
1986
1987/**
1988 * i915_gem_get_ggtt_alignment - return required global GTT alignment
1989 * @dev_priv: i915 device
1990 * @size: object size
1991 * @tiling_mode: tiling mode
1992 * @fenced: is fenced alignment required or not
1993 *
1994 * Return the required global GTT alignment for an object, taking into account
1995 * potential fence register mapping.
1996 */
1997u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
1998                                int tiling_mode, bool fenced)
1999{
2000        GEM_BUG_ON(size == 0);
2001
2002        /*
2003         * Minimum alignment is 4k (GTT page size), but might be greater
2004         * if a fence register is needed for the object.
2005         */
2006        if (INTEL_GEN(dev_priv) >= 4 || (!fenced && IS_G33(dev_priv)) ||
2007            tiling_mode == I915_TILING_NONE)
2008                return 4096;
2009
2010        /*
2011         * Previous chips need to be aligned to the size of the smallest
2012         * fence register that can contain the object.
2013         */
2014        return i915_gem_get_ggtt_size(dev_priv, size, tiling_mode);
2015}
2016
2017static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2018{
2019        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2020        int err;
2021
2022        err = drm_gem_create_mmap_offset(&obj->base);
2023        if (!err)
2024                return 0;
2025
2026        /* We can idle the GPU locklessly to flush stale objects, but in order
2027         * to claim that space for ourselves, we need to take the big
2028         * struct_mutex to free the requests+objects and allocate our slot.
2029         */
2030        err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
2031        if (err)
2032                return err;
2033
2034        err = i915_mutex_lock_interruptible(&dev_priv->drm);
2035        if (!err) {
2036                i915_gem_retire_requests(dev_priv);
2037                err = drm_gem_create_mmap_offset(&obj->base);
2038                mutex_unlock(&dev_priv->drm.struct_mutex);
2039        }
2040
2041        return err;
2042}
2043
2044static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2045{
2046        drm_gem_free_mmap_offset(&obj->base);
2047}
2048
2049int
2050i915_gem_mmap_gtt(struct drm_file *file,
2051                  struct drm_device *dev,
2052                  uint32_t handle,
2053                  uint64_t *offset)
2054{
2055        struct drm_i915_gem_object *obj;
2056        int ret;
2057
2058        obj = i915_gem_object_lookup(file, handle);
2059        if (!obj)
2060                return -ENOENT;
2061
2062        ret = i915_gem_object_create_mmap_offset(obj);
2063        if (ret == 0)
2064                *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2065
2066        i915_gem_object_put_unlocked(obj);
2067        return ret;
2068}
2069
2070/**
2071 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2072 * @dev: DRM device
2073 * @data: GTT mapping ioctl data
2074 * @file: GEM object info
2075 *
2076 * Simply returns the fake offset to userspace so it can mmap it.
2077 * The mmap call will end up in drm_gem_mmap(), which will set things
2078 * up so we can get faults in the handler above.
2079 *
2080 * The fault handler will take care of binding the object into the GTT
2081 * (since it may have been evicted to make room for something), allocating
2082 * a fence register, and mapping the appropriate aperture address into
2083 * userspace.
2084 */
2085int
2086i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2087                        struct drm_file *file)
2088{
2089        struct drm_i915_gem_mmap_gtt *args = data;
2090
2091        return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2092}
2093
2094/* Immediately discard the backing storage */
2095static void
2096i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2097{
2098        i915_gem_object_free_mmap_offset(obj);
2099
2100        if (obj->base.filp == NULL)
2101                return;
2102
2103        /* Our goal here is to return as much of the memory as
2104         * is possible back to the system as we are called from OOM.
2105         * To do this we must instruct the shmfs to drop all of its
2106         * backing pages, *now*.
2107         */
2108        shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2109        obj->madv = __I915_MADV_PURGED;
2110}
2111
2112/* Try to discard unwanted pages */
2113static void
2114i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2115{
2116        struct address_space *mapping;
2117
2118        switch (obj->madv) {
2119        case I915_MADV_DONTNEED:
2120                i915_gem_object_truncate(obj);
2121        case __I915_MADV_PURGED:
2122                return;
2123        }
2124
2125        if (obj->base.filp == NULL)
2126                return;
2127
2128        mapping = obj->base.filp->f_mapping,
2129        invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2130}
2131
2132static void
2133i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2134{
2135        struct sgt_iter sgt_iter;
2136        struct page *page;
2137        int ret;
2138
2139        BUG_ON(obj->madv == __I915_MADV_PURGED);
2140
2141        ret = i915_gem_object_set_to_cpu_domain(obj, true);
2142        if (WARN_ON(ret)) {
2143                /* In the event of a disaster, abandon all caches and
2144                 * hope for the best.
2145                 */
2146                i915_gem_clflush_object(obj, true);
2147                obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2148        }
2149
2150        i915_gem_gtt_finish_object(obj);
2151
2152        if (i915_gem_object_needs_bit17_swizzle(obj))
2153                i915_gem_object_save_bit_17_swizzle(obj);
2154
2155        if (obj->madv == I915_MADV_DONTNEED)
2156                obj->dirty = 0;
2157
2158        for_each_sgt_page(page, sgt_iter, obj->pages) {
2159                if (obj->dirty)
2160                        set_page_dirty(page);
2161
2162                if (obj->madv == I915_MADV_WILLNEED)
2163                        mark_page_accessed(page);
2164
2165                put_page(page);
2166        }
2167        obj->dirty = 0;
2168
2169        sg_free_table(obj->pages);
2170        kfree(obj->pages);
2171}
2172
2173int
2174i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2175{
2176        const struct drm_i915_gem_object_ops *ops = obj->ops;
2177
2178        if (obj->pages == NULL)
2179                return 0;
2180
2181        if (obj->pages_pin_count)
2182                return -EBUSY;
2183
2184        GEM_BUG_ON(obj->bind_count);
2185
2186        /* ->put_pages might need to allocate memory for the bit17 swizzle
2187         * array, hence protect them from being reaped by removing them from gtt
2188         * lists early. */
2189        list_del(&obj->global_list);
2190
2191        if (obj->mapping) {
2192                void *ptr;
2193
2194                ptr = ptr_mask_bits(obj->mapping);
2195                if (is_vmalloc_addr(ptr))
2196                        vunmap(ptr);
2197                else
2198                        kunmap(kmap_to_page(ptr));
2199
2200                obj->mapping = NULL;
2201        }
2202
2203        ops->put_pages(obj);
2204        obj->pages = NULL;
2205
2206        i915_gem_object_invalidate(obj);
2207
2208        return 0;
2209}
2210
2211static int
2212i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2213{
2214        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2215        int page_count, i;
2216        struct address_space *mapping;
2217        struct sg_table *st;
2218        struct scatterlist *sg;
2219        struct sgt_iter sgt_iter;
2220        struct page *page;
2221        unsigned long last_pfn = 0;     /* suppress gcc warning */
2222        int ret;
2223        gfp_t gfp;
2224
2225        /* Assert that the object is not currently in any GPU domain. As it
2226         * wasn't in the GTT, there shouldn't be any way it could have been in
2227         * a GPU cache
2228         */
2229        BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2230        BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2231
2232        st = kmalloc(sizeof(*st), GFP_KERNEL);
2233        if (st == NULL)
2234                return -ENOMEM;
2235
2236        page_count = obj->base.size / PAGE_SIZE;
2237        if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2238                kfree(st);
2239                return -ENOMEM;
2240        }
2241
2242        /* Get the list of pages out of our struct file.  They'll be pinned
2243         * at this point until we release them.
2244         *
2245         * Fail silently without starting the shrinker
2246         */
2247        mapping = obj->base.filp->f_mapping;
2248        gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2249        gfp |= __GFP_NORETRY | __GFP_NOWARN;
2250        sg = st->sgl;
2251        st->nents = 0;
2252        for (i = 0; i < page_count; i++) {
2253                page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2254                if (IS_ERR(page)) {
2255                        i915_gem_shrink(dev_priv,
2256                                        page_count,
2257                                        I915_SHRINK_BOUND |
2258                                        I915_SHRINK_UNBOUND |
2259                                        I915_SHRINK_PURGEABLE);
2260                        page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2261                }
2262                if (IS_ERR(page)) {
2263                        /* We've tried hard to allocate the memory by reaping
2264                         * our own buffer, now let the real VM do its job and
2265                         * go down in flames if truly OOM.
2266                         */
2267                        i915_gem_shrink_all(dev_priv);
2268                        page = shmem_read_mapping_page(mapping, i);
2269                        if (IS_ERR(page)) {
2270                                ret = PTR_ERR(page);
2271                                goto err_sg;
2272                        }
2273                }
2274#ifdef CONFIG_SWIOTLB
2275                if (swiotlb_nr_tbl()) {
2276                        st->nents++;
2277                        sg_set_page(sg, page, PAGE_SIZE, 0);
2278                        sg = sg_next(sg);
2279                        continue;
2280                }
2281#endif
2282                if (!i || page_to_pfn(page) != last_pfn + 1) {
2283                        if (i)
2284                                sg = sg_next(sg);
2285                        st->nents++;
2286                        sg_set_page(sg, page, PAGE_SIZE, 0);
2287                } else {
2288                        sg->length += PAGE_SIZE;
2289                }
2290                last_pfn = page_to_pfn(page);
2291
2292                /* Check that the i965g/gm workaround works. */
2293                WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2294        }
2295#ifdef CONFIG_SWIOTLB
2296        if (!swiotlb_nr_tbl())
2297#endif
2298                sg_mark_end(sg);
2299        obj->pages = st;
2300
2301        ret = i915_gem_gtt_prepare_object(obj);
2302        if (ret)
2303                goto err_pages;
2304
2305        if (i915_gem_object_needs_bit17_swizzle(obj))
2306                i915_gem_object_do_bit_17_swizzle(obj);
2307
2308        if (i915_gem_object_is_tiled(obj) &&
2309            dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2310                i915_gem_object_pin_pages(obj);
2311
2312        return 0;
2313
2314err_sg:
2315        sg_mark_end(sg);
2316err_pages:
2317        for_each_sgt_page(page, sgt_iter, st)
2318                put_page(page);
2319        sg_free_table(st);
2320        kfree(st);
2321
2322        /* shmemfs first checks if there is enough memory to allocate the page
2323         * and reports ENOSPC should there be insufficient, along with the usual
2324         * ENOMEM for a genuine allocation failure.
2325         *
2326         * We use ENOSPC in our driver to mean that we have run out of aperture
2327         * space and so want to translate the error from shmemfs back to our
2328         * usual understanding of ENOMEM.
2329         */
2330        if (ret == -ENOSPC)
2331                ret = -ENOMEM;
2332
2333        return ret;
2334}
2335
2336/* Ensure that the associated pages are gathered from the backing storage
2337 * and pinned into our object. i915_gem_object_get_pages() may be called
2338 * multiple times before they are released by a single call to
2339 * i915_gem_object_put_pages() - once the pages are no longer referenced
2340 * either as a result of memory pressure (reaping pages under the shrinker)
2341 * or as the object is itself released.
2342 */
2343int
2344i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2345{
2346        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2347        const struct drm_i915_gem_object_ops *ops = obj->ops;
2348        int ret;
2349
2350        if (obj->pages)
2351                return 0;
2352
2353        if (obj->madv != I915_MADV_WILLNEED) {
2354                DRM_DEBUG("Attempting to obtain a purgeable object\n");
2355                return -EFAULT;
2356        }
2357
2358        BUG_ON(obj->pages_pin_count);
2359
2360        ret = ops->get_pages(obj);
2361        if (ret)
2362                return ret;
2363
2364        list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2365
2366        obj->get_page.sg = obj->pages->sgl;
2367        obj->get_page.last = 0;
2368
2369        return 0;
2370}
2371
2372/* The 'mapping' part of i915_gem_object_pin_map() below */
2373static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2374                                 enum i915_map_type type)
2375{
2376        unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2377        struct sg_table *sgt = obj->pages;
2378        struct sgt_iter sgt_iter;
2379        struct page *page;
2380        struct page *stack_pages[32];
2381        struct page **pages = stack_pages;
2382        unsigned long i = 0;
2383        pgprot_t pgprot;
2384        void *addr;
2385
2386        /* A single page can always be kmapped */
2387        if (n_pages == 1 && type == I915_MAP_WB)
2388                return kmap(sg_page(sgt->sgl));
2389
2390        if (n_pages > ARRAY_SIZE(stack_pages)) {
2391                /* Too big for stack -- allocate temporary array instead */
2392                pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2393                if (!pages)
2394                        return NULL;
2395        }
2396
2397        for_each_sgt_page(page, sgt_iter, sgt)
2398                pages[i++] = page;
2399
2400        /* Check that we have the expected number of pages */
2401        GEM_BUG_ON(i != n_pages);
2402
2403        switch (type) {
2404        case I915_MAP_WB:
2405                pgprot = PAGE_KERNEL;
2406                break;
2407        case I915_MAP_WC:
2408                pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2409                break;
2410        }
2411        addr = vmap(pages, n_pages, 0, pgprot);
2412
2413        if (pages != stack_pages)
2414                drm_free_large(pages);
2415
2416        return addr;
2417}
2418
2419/* get, pin, and map the pages of the object into kernel space */
2420void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2421                              enum i915_map_type type)
2422{
2423        enum i915_map_type has_type;
2424        bool pinned;
2425        void *ptr;
2426        int ret;
2427
2428        lockdep_assert_held(&obj->base.dev->struct_mutex);
2429        GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
2430
2431        ret = i915_gem_object_get_pages(obj);
2432        if (ret)
2433                return ERR_PTR(ret);
2434
2435        i915_gem_object_pin_pages(obj);
2436        pinned = obj->pages_pin_count > 1;
2437
2438        ptr = ptr_unpack_bits(obj->mapping, has_type);
2439        if (ptr && has_type != type) {
2440                if (pinned) {
2441                        ret = -EBUSY;
2442                        goto err;
2443                }
2444
2445                if (is_vmalloc_addr(ptr))
2446                        vunmap(ptr);
2447                else
2448                        kunmap(kmap_to_page(ptr));
2449
2450                ptr = obj->mapping = NULL;
2451        }
2452
2453        if (!ptr) {
2454                ptr = i915_gem_object_map(obj, type);
2455                if (!ptr) {
2456                        ret = -ENOMEM;
2457                        goto err;
2458                }
2459
2460                obj->mapping = ptr_pack_bits(ptr, type);
2461        }
2462
2463        return ptr;
2464
2465err:
2466        i915_gem_object_unpin_pages(obj);
2467        return ERR_PTR(ret);
2468}
2469
2470static void
2471i915_gem_object_retire__write(struct i915_gem_active *active,
2472                              struct drm_i915_gem_request *request)
2473{
2474        struct drm_i915_gem_object *obj =
2475                container_of(active, struct drm_i915_gem_object, last_write);
2476
2477        intel_fb_obj_flush(obj, true, ORIGIN_CS);
2478}
2479
2480static void
2481i915_gem_object_retire__read(struct i915_gem_active *active,
2482                             struct drm_i915_gem_request *request)
2483{
2484        int idx = request->engine->id;
2485        struct drm_i915_gem_object *obj =
2486                container_of(active, struct drm_i915_gem_object, last_read[idx]);
2487
2488        GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx));
2489
2490        i915_gem_object_clear_active(obj, idx);
2491        if (i915_gem_object_is_active(obj))
2492                return;
2493
2494        /* Bump our place on the bound list to keep it roughly in LRU order
2495         * so that we don't steal from recently used but inactive objects
2496         * (unless we are forced to ofc!)
2497         */
2498        if (obj->bind_count)
2499                list_move_tail(&obj->global_list,
2500                               &request->i915->mm.bound_list);
2501
2502        i915_gem_object_put(obj);
2503}
2504
2505static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2506{
2507        unsigned long elapsed;
2508
2509        if (ctx->hang_stats.banned)
2510                return true;
2511
2512        elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2513        if (ctx->hang_stats.ban_period_seconds &&
2514            elapsed <= ctx->hang_stats.ban_period_seconds) {
2515                DRM_DEBUG("context hanging too fast, banning!\n");
2516                return true;
2517        }
2518
2519        return false;
2520}
2521
2522static void i915_set_reset_status(struct i915_gem_context *ctx,
2523                                  const bool guilty)
2524{
2525        struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2526
2527        if (guilty) {
2528                hs->banned = i915_context_is_banned(ctx);
2529                hs->batch_active++;
2530                hs->guilty_ts = get_seconds();
2531        } else {
2532                hs->batch_pending++;
2533        }
2534}
2535
2536struct drm_i915_gem_request *
2537i915_gem_find_active_request(struct intel_engine_cs *engine)
2538{
2539        struct drm_i915_gem_request *request;
2540
2541        /* We are called by the error capture and reset at a random
2542         * point in time. In particular, note that neither is crucially
2543         * ordered with an interrupt. After a hang, the GPU is dead and we
2544         * assume that no more writes can happen (we waited long enough for
2545         * all writes that were in transaction to be flushed) - adding an
2546         * extra delay for a recent interrupt is pointless. Hence, we do
2547         * not need an engine->irq_seqno_barrier() before the seqno reads.
2548         */
2549        list_for_each_entry(request, &engine->request_list, link) {
2550                if (i915_gem_request_completed(request))
2551                        continue;
2552
2553                if (!i915_sw_fence_done(&request->submit))
2554                        break;
2555
2556                return request;
2557        }
2558
2559        return NULL;
2560}
2561
2562static void reset_request(struct drm_i915_gem_request *request)
2563{
2564        void *vaddr = request->ring->vaddr;
2565        u32 head;
2566
2567        /* As this request likely depends on state from the lost
2568         * context, clear out all the user operations leaving the
2569         * breadcrumb at the end (so we get the fence notifications).
2570         */
2571        head = request->head;
2572        if (request->postfix < head) {
2573                memset(vaddr + head, 0, request->ring->size - head);
2574                head = 0;
2575        }
2576        memset(vaddr + head, 0, request->postfix - head);
2577}
2578
2579static void i915_gem_reset_engine(struct intel_engine_cs *engine)
2580{
2581        struct drm_i915_gem_request *request;
2582        struct i915_gem_context *incomplete_ctx;
2583        bool ring_hung;
2584
2585        /* Ensure irq handler finishes, and not run again. */
2586        tasklet_kill(&engine->irq_tasklet);
2587        if (engine->irq_seqno_barrier)
2588                engine->irq_seqno_barrier(engine);
2589
2590        request = i915_gem_find_active_request(engine);
2591        if (!request)
2592                return;
2593
2594        ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2595        i915_set_reset_status(request->ctx, ring_hung);
2596        if (!ring_hung)
2597                return;
2598
2599        DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
2600                         engine->name, request->fence.seqno);
2601
2602        /* Setup the CS to resume from the breadcrumb of the hung request */
2603        engine->reset_hw(engine, request);
2604
2605        /* Users of the default context do not rely on logical state
2606         * preserved between batches. They have to emit full state on
2607         * every batch and so it is safe to execute queued requests following
2608         * the hang.
2609         *
2610         * Other contexts preserve state, now corrupt. We want to skip all
2611         * queued requests that reference the corrupt context.
2612         */
2613        incomplete_ctx = request->ctx;
2614        if (i915_gem_context_is_default(incomplete_ctx))
2615                return;
2616
2617        list_for_each_entry_continue(request, &engine->request_list, link)
2618                if (request->ctx == incomplete_ctx)
2619                        reset_request(request);
2620}
2621
2622void i915_gem_reset(struct drm_i915_private *dev_priv)
2623{
2624        struct intel_engine_cs *engine;
2625
2626        i915_gem_retire_requests(dev_priv);
2627
2628        for_each_engine(engine, dev_priv)
2629                i915_gem_reset_engine(engine);
2630
2631        i915_gem_restore_fences(&dev_priv->drm);
2632
2633        if (dev_priv->gt.awake) {
2634                intel_sanitize_gt_powersave(dev_priv);
2635                intel_enable_gt_powersave(dev_priv);
2636                if (INTEL_GEN(dev_priv) >= 6)
2637                        gen6_rps_busy(dev_priv);
2638        }
2639}
2640
2641static void nop_submit_request(struct drm_i915_gem_request *request)
2642{
2643}
2644
2645static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
2646{
2647        engine->submit_request = nop_submit_request;
2648
2649        /* Mark all pending requests as complete so that any concurrent
2650         * (lockless) lookup doesn't try and wait upon the request as we
2651         * reset it.
2652         */
2653        intel_engine_init_seqno(engine, engine->last_submitted_seqno);
2654
2655        /*
2656         * Clear the execlists queue up before freeing the requests, as those
2657         * are the ones that keep the context and ringbuffer backing objects
2658         * pinned in place.
2659         */
2660
2661        if (i915.enable_execlists) {
2662                spin_lock(&engine->execlist_lock);
2663                INIT_LIST_HEAD(&engine->execlist_queue);
2664                i915_gem_request_put(engine->execlist_port[0].request);
2665                i915_gem_request_put(engine->execlist_port[1].request);
2666                memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
2667                spin_unlock(&engine->execlist_lock);
2668        }
2669
2670        engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
2671}
2672
2673void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
2674{
2675        struct intel_engine_cs *engine;
2676
2677        lockdep_assert_held(&dev_priv->drm.struct_mutex);
2678        set_bit(I915_WEDGED, &dev_priv->gpu_error.flags);
2679
2680        i915_gem_context_lost(dev_priv);
2681        for_each_engine(engine, dev_priv)
2682                i915_gem_cleanup_engine(engine);
2683        mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2684
2685        i915_gem_retire_requests(dev_priv);
2686}
2687
2688static void
2689i915_gem_retire_work_handler(struct work_struct *work)
2690{
2691        struct drm_i915_private *dev_priv =
2692                container_of(work, typeof(*dev_priv), gt.retire_work.work);
2693        struct drm_device *dev = &dev_priv->drm;
2694
2695        /* Come back later if the device is busy... */
2696        if (mutex_trylock(&dev->struct_mutex)) {
2697                i915_gem_retire_requests(dev_priv);
2698                mutex_unlock(&dev->struct_mutex);
2699        }
2700
2701        /* Keep the retire handler running until we are finally idle.
2702         * We do not need to do this test under locking as in the worst-case
2703         * we queue the retire worker once too often.
2704         */
2705        if (READ_ONCE(dev_priv->gt.awake)) {
2706                i915_queue_hangcheck(dev_priv);
2707                queue_delayed_work(dev_priv->wq,
2708                                   &dev_priv->gt.retire_work,
2709                                   round_jiffies_up_relative(HZ));
2710        }
2711}
2712
2713static void
2714i915_gem_idle_work_handler(struct work_struct *work)
2715{
2716        struct drm_i915_private *dev_priv =
2717                container_of(work, typeof(*dev_priv), gt.idle_work.work);
2718        struct drm_device *dev = &dev_priv->drm;
2719        struct intel_engine_cs *engine;
2720        bool rearm_hangcheck;
2721
2722        if (!READ_ONCE(dev_priv->gt.awake))
2723                return;
2724
2725        if (READ_ONCE(dev_priv->gt.active_engines))
2726                return;
2727
2728        rearm_hangcheck =
2729                cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2730
2731        if (!mutex_trylock(&dev->struct_mutex)) {
2732                /* Currently busy, come back later */
2733                mod_delayed_work(dev_priv->wq,
2734                                 &dev_priv->gt.idle_work,
2735                                 msecs_to_jiffies(50));
2736                goto out_rearm;
2737        }
2738
2739        if (dev_priv->gt.active_engines)
2740                goto out_unlock;
2741
2742        for_each_engine(engine, dev_priv)
2743                i915_gem_batch_pool_fini(&engine->batch_pool);
2744
2745        GEM_BUG_ON(!dev_priv->gt.awake);
2746        dev_priv->gt.awake = false;
2747        rearm_hangcheck = false;
2748
2749        if (INTEL_GEN(dev_priv) >= 6)
2750                gen6_rps_idle(dev_priv);
2751        intel_runtime_pm_put(dev_priv);
2752out_unlock:
2753        mutex_unlock(&dev->struct_mutex);
2754
2755out_rearm:
2756        if (rearm_hangcheck) {
2757                GEM_BUG_ON(!dev_priv->gt.awake);
2758                i915_queue_hangcheck(dev_priv);
2759        }
2760}
2761
2762void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
2763{
2764        struct drm_i915_gem_object *obj = to_intel_bo(gem);
2765        struct drm_i915_file_private *fpriv = file->driver_priv;
2766        struct i915_vma *vma, *vn;
2767
2768        mutex_lock(&obj->base.dev->struct_mutex);
2769        list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
2770                if (vma->vm->file == fpriv)
2771                        i915_vma_close(vma);
2772        mutex_unlock(&obj->base.dev->struct_mutex);
2773}
2774
2775/**
2776 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2777 * @dev: drm device pointer
2778 * @data: ioctl data blob
2779 * @file: drm file pointer
2780 *
2781 * Returns 0 if successful, else an error is returned with the remaining time in
2782 * the timeout parameter.
2783 *  -ETIME: object is still busy after timeout
2784 *  -ERESTARTSYS: signal interrupted the wait
2785 *  -ENONENT: object doesn't exist
2786 * Also possible, but rare:
2787 *  -EAGAIN: GPU wedged
2788 *  -ENOMEM: damn
2789 *  -ENODEV: Internal IRQ fail
2790 *  -E?: The add request failed
2791 *
2792 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2793 * non-zero timeout parameter the wait ioctl will wait for the given number of
2794 * nanoseconds on an object becoming unbusy. Since the wait itself does so
2795 * without holding struct_mutex the object may become re-busied before this
2796 * function completes. A similar but shorter * race condition exists in the busy
2797 * ioctl
2798 */
2799int
2800i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2801{
2802        struct drm_i915_gem_wait *args = data;
2803        struct intel_rps_client *rps = to_rps_client(file);
2804        struct drm_i915_gem_object *obj;
2805        unsigned long active;
2806        int idx, ret = 0;
2807
2808        if (args->flags != 0)
2809                return -EINVAL;
2810
2811        obj = i915_gem_object_lookup(file, args->bo_handle);
2812        if (!obj)
2813                return -ENOENT;
2814
2815        active = __I915_BO_ACTIVE(obj);
2816        for_each_active(active, idx) {
2817                s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL;
2818                ret = i915_gem_active_wait_unlocked(&obj->last_read[idx],
2819                                                    I915_WAIT_INTERRUPTIBLE,
2820                                                    timeout, rps);
2821                if (ret)
2822                        break;
2823        }
2824
2825        i915_gem_object_put_unlocked(obj);
2826        return ret;
2827}
2828
2829static void __i915_vma_iounmap(struct i915_vma *vma)
2830{
2831        GEM_BUG_ON(i915_vma_is_pinned(vma));
2832
2833        if (vma->iomap == NULL)
2834                return;
2835
2836        io_mapping_unmap(vma->iomap);
2837        vma->iomap = NULL;
2838}
2839
2840int i915_vma_unbind(struct i915_vma *vma)
2841{
2842        struct drm_i915_gem_object *obj = vma->obj;
2843        unsigned long active;
2844        int ret;
2845
2846        /* First wait upon any activity as retiring the request may
2847         * have side-effects such as unpinning or even unbinding this vma.
2848         */
2849        active = i915_vma_get_active(vma);
2850        if (active) {
2851                int idx;
2852
2853                /* When a closed VMA is retired, it is unbound - eek.
2854                 * In order to prevent it from being recursively closed,
2855                 * take a pin on the vma so that the second unbind is
2856                 * aborted.
2857                 */
2858                __i915_vma_pin(vma);
2859
2860                for_each_active(active, idx) {
2861                        ret = i915_gem_active_retire(&vma->last_read[idx],
2862                                                   &vma->vm->dev->struct_mutex);
2863                        if (ret)
2864                                break;
2865                }
2866
2867                __i915_vma_unpin(vma);
2868                if (ret)
2869                        return ret;
2870
2871                GEM_BUG_ON(i915_vma_is_active(vma));
2872        }
2873
2874        if (i915_vma_is_pinned(vma))
2875                return -EBUSY;
2876
2877        if (!drm_mm_node_allocated(&vma->node))
2878                goto destroy;
2879
2880        GEM_BUG_ON(obj->bind_count == 0);
2881        GEM_BUG_ON(!obj->pages);
2882
2883        if (i915_vma_is_map_and_fenceable(vma)) {
2884                /* release the fence reg _after_ flushing */
2885                ret = i915_vma_put_fence(vma);
2886                if (ret)
2887                        return ret;
2888
2889                /* Force a pagefault for domain tracking on next user access */
2890                i915_gem_release_mmap(obj);
2891
2892                __i915_vma_iounmap(vma);
2893                vma->flags &= ~I915_VMA_CAN_FENCE;
2894        }
2895
2896        if (likely(!vma->vm->closed)) {
2897                trace_i915_vma_unbind(vma);
2898                vma->vm->unbind_vma(vma);
2899        }
2900        vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
2901
2902        drm_mm_remove_node(&vma->node);
2903        list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
2904
2905        if (vma->pages != obj->pages) {
2906                GEM_BUG_ON(!vma->pages);
2907                sg_free_table(vma->pages);
2908                kfree(vma->pages);
2909        }
2910        vma->pages = NULL;
2911
2912        /* Since the unbound list is global, only move to that list if
2913         * no more VMAs exist. */
2914        if (--obj->bind_count == 0)
2915                list_move_tail(&obj->global_list,
2916                               &to_i915(obj->base.dev)->mm.unbound_list);
2917
2918        /* And finally now the object is completely decoupled from this vma,
2919         * we can drop its hold on the backing storage and allow it to be
2920         * reaped by the shrinker.
2921         */
2922        i915_gem_object_unpin_pages(obj);
2923
2924destroy:
2925        if (unlikely(i915_vma_is_closed(vma)))
2926                i915_vma_destroy(vma);
2927
2928        return 0;
2929}
2930
2931int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
2932                           unsigned int flags)
2933{
2934        struct intel_engine_cs *engine;
2935        int ret;
2936
2937        for_each_engine(engine, dev_priv) {
2938                if (engine->last_context == NULL)
2939                        continue;
2940
2941                ret = intel_engine_idle(engine, flags);
2942                if (ret)
2943                        return ret;
2944        }
2945
2946        return 0;
2947}
2948
2949static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
2950                                     unsigned long cache_level)
2951{
2952        struct drm_mm_node *gtt_space = &vma->node;
2953        struct drm_mm_node *other;
2954
2955        /*
2956         * On some machines we have to be careful when putting differing types
2957         * of snoopable memory together to avoid the prefetcher crossing memory
2958         * domains and dying. During vm initialisation, we decide whether or not
2959         * these constraints apply and set the drm_mm.color_adjust
2960         * appropriately.
2961         */
2962        if (vma->vm->mm.color_adjust == NULL)
2963                return true;
2964
2965        if (!drm_mm_node_allocated(gtt_space))
2966                return true;
2967
2968        if (list_empty(&gtt_space->node_list))
2969                return true;
2970
2971        other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2972        if (other->allocated && !other->hole_follows && other->color != cache_level)
2973                return false;
2974
2975        other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2976        if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2977                return false;
2978
2979        return true;
2980}
2981
2982/**
2983 * i915_vma_insert - finds a slot for the vma in its address space
2984 * @vma: the vma
2985 * @size: requested size in bytes (can be larger than the VMA)
2986 * @alignment: required alignment
2987 * @flags: mask of PIN_* flags to use
2988 *
2989 * First we try to allocate some free space that meets the requirements for
2990 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
2991 * preferrably the oldest idle entry to make room for the new VMA.
2992 *
2993 * Returns:
2994 * 0 on success, negative error code otherwise.
2995 */
2996static int
2997i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
2998{
2999        struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
3000        struct drm_i915_gem_object *obj = vma->obj;
3001        u64 start, end;
3002        int ret;
3003
3004        GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
3005        GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
3006
3007        size = max(size, vma->size);
3008        if (flags & PIN_MAPPABLE)
3009                size = i915_gem_get_ggtt_size(dev_priv, size,
3010                                              i915_gem_object_get_tiling(obj));
3011
3012        alignment = max(max(alignment, vma->display_alignment),
3013                        i915_gem_get_ggtt_alignment(dev_priv, size,
3014                                                    i915_gem_object_get_tiling(obj),
3015                                                    flags & PIN_MAPPABLE));
3016
3017        start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3018
3019        end = vma->vm->total;
3020        if (flags & PIN_MAPPABLE)
3021                end = min_t(u64, end, dev_priv->ggtt.mappable_end);
3022        if (flags & PIN_ZONE_4G)
3023                end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3024
3025        /* If binding the object/GGTT view requires more space than the entire
3026         * aperture has, reject it early before evicting everything in a vain
3027         * attempt to find space.
3028         */
3029        if (size > end) {
3030                DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
3031                          size, obj->base.size,
3032                          flags & PIN_MAPPABLE ? "mappable" : "total",
3033                          end);
3034                return -E2BIG;
3035        }
3036
3037        ret = i915_gem_object_get_pages(obj);
3038        if (ret)
3039                return ret;
3040
3041        i915_gem_object_pin_pages(obj);
3042
3043        if (flags & PIN_OFFSET_FIXED) {
3044                u64 offset = flags & PIN_OFFSET_MASK;
3045                if (offset & (alignment - 1) || offset > end - size) {
3046                        ret = -EINVAL;
3047                        goto err_unpin;
3048                }
3049
3050                vma->node.start = offset;
3051                vma->node.size = size;
3052                vma->node.color = obj->cache_level;
3053                ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
3054                if (ret) {
3055                        ret = i915_gem_evict_for_vma(vma);
3056                        if (ret == 0)
3057                                ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
3058                        if (ret)
3059                                goto err_unpin;
3060                }
3061        } else {
3062                u32 search_flag, alloc_flag;
3063
3064                if (flags & PIN_HIGH) {
3065                        search_flag = DRM_MM_SEARCH_BELOW;
3066                        alloc_flag = DRM_MM_CREATE_TOP;
3067                } else {
3068                        search_flag = DRM_MM_SEARCH_DEFAULT;
3069                        alloc_flag = DRM_MM_CREATE_DEFAULT;
3070                }
3071
3072                /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3073                 * so we know that we always have a minimum alignment of 4096.
3074                 * The drm_mm range manager is optimised to return results
3075                 * with zero alignment, so where possible use the optimal
3076                 * path.
3077                 */
3078                if (alignment <= 4096)
3079                        alignment = 0;
3080
3081search_free:
3082                ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
3083                                                          &vma->node,
3084                                                          size, alignment,
3085                                                          obj->cache_level,
3086                                                          start, end,
3087                                                          search_flag,
3088                                                          alloc_flag);
3089                if (ret) {
3090                        ret = i915_gem_evict_something(vma->vm, size, alignment,
3091                                                       obj->cache_level,
3092                                                       start, end,
3093                                                       flags);
3094                        if (ret == 0)
3095                                goto search_free;
3096
3097                        goto err_unpin;
3098                }
3099        }
3100        GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
3101
3102        list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3103        list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3104        obj->bind_count++;
3105
3106        return 0;
3107
3108err_unpin:
3109        i915_gem_object_unpin_pages(obj);
3110        return ret;
3111}
3112
3113bool
3114i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3115                        bool force)
3116{
3117        /* If we don't have a page list set up, then we're not pinned
3118         * to GPU, and we can ignore the cache flush because it'll happen
3119         * again at bind time.
3120         */
3121        if (obj->pages == NULL)
3122                return false;
3123
3124        /*
3125         * Stolen memory is always coherent with the GPU as it is explicitly
3126         * marked as wc by the system, or the system is cache-coherent.
3127         */
3128        if (obj->stolen || obj->phys_handle)
3129                return false;
3130
3131        /* If the GPU is snooping the contents of the CPU cache,
3132         * we do not need to manually clear the CPU cache lines.  However,
3133         * the caches are only snooped when the render cache is
3134         * flushed/invalidated.  As we always have to emit invalidations
3135         * and flushes when moving into and out of the RENDER domain, correct
3136         * snooping behaviour occurs naturally as the result of our domain
3137         * tracking.
3138         */
3139        if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3140                obj->cache_dirty = true;
3141                return false;
3142        }
3143
3144        trace_i915_gem_object_clflush(obj);
3145        drm_clflush_sg(obj->pages);
3146        obj->cache_dirty = false;
3147
3148        return true;
3149}
3150
3151/** Flushes the GTT write domain for the object if it's dirty. */
3152static void
3153i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3154{
3155        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3156
3157        if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3158                return;
3159
3160        /* No actual flushing is required for the GTT write domain.  Writes
3161         * to it "immediately" go to main memory as far as we know, so there's
3162         * no chipset flush.  It also doesn't land in render cache.
3163         *
3164         * However, we do have to enforce the order so that all writes through
3165         * the GTT land before any writes to the device, such as updates to
3166         * the GATT itself.
3167         *
3168         * We also have to wait a bit for the writes to land from the GTT.
3169         * An uncached read (i.e. mmio) seems to be ideal for the round-trip
3170         * timing. This issue has only been observed when switching quickly
3171         * between GTT writes and CPU reads from inside the kernel on recent hw,
3172         * and it appears to only affect discrete GTT blocks (i.e. on LLC
3173         * system agents we cannot reproduce this behaviour).
3174         */
3175        wmb();
3176        if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv))
3177                POSTING_READ(RING_ACTHD(dev_priv->engine[RCS].mmio_base));
3178
3179        intel_fb_obj_flush(obj, false, write_origin(obj, I915_GEM_DOMAIN_GTT));
3180
3181        obj->base.write_domain = 0;
3182        trace_i915_gem_object_change_domain(obj,
3183                                            obj->base.read_domains,
3184                                            I915_GEM_DOMAIN_GTT);
3185}
3186
3187/** Flushes the CPU write domain for the object if it's dirty. */
3188static void
3189i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3190{
3191        if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3192                return;
3193
3194        if (i915_gem_clflush_object(obj, obj->pin_display))
3195                i915_gem_chipset_flush(to_i915(obj->base.dev));
3196
3197        intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3198
3199        obj->base.write_domain = 0;
3200        trace_i915_gem_object_change_domain(obj,
3201                                            obj->base.read_domains,
3202                                            I915_GEM_DOMAIN_CPU);
3203}
3204
3205static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
3206{
3207        struct i915_vma *vma;
3208
3209        list_for_each_entry(vma, &obj->vma_list, obj_link) {
3210                if (!i915_vma_is_ggtt(vma))
3211                        continue;
3212
3213                if (i915_vma_is_active(vma))
3214                        continue;
3215
3216                if (!drm_mm_node_allocated(&vma->node))
3217                        continue;
3218
3219                list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3220        }
3221}
3222
3223/**
3224 * Moves a single object to the GTT read, and possibly write domain.
3225 * @obj: object to act on
3226 * @write: ask for write access or read only
3227 *
3228 * This function returns when the move is complete, including waiting on
3229 * flushes to occur.
3230 */
3231int
3232i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3233{
3234        uint32_t old_write_domain, old_read_domains;
3235        int ret;
3236
3237        ret = i915_gem_object_wait_rendering(obj, !write);
3238        if (ret)
3239                return ret;
3240
3241        if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3242                return 0;
3243
3244        /* Flush and acquire obj->pages so that we are coherent through
3245         * direct access in memory with previous cached writes through
3246         * shmemfs and that our cache domain tracking remains valid.
3247         * For example, if the obj->filp was moved to swap without us
3248         * being notified and releasing the pages, we would mistakenly
3249         * continue to assume that the obj remained out of the CPU cached
3250         * domain.
3251         */
3252        ret = i915_gem_object_get_pages(obj);
3253        if (ret)
3254                return ret;
3255
3256        i915_gem_object_flush_cpu_write_domain(obj);
3257
3258        /* Serialise direct access to this object with the barriers for
3259         * coherent writes from the GPU, by effectively invalidating the
3260         * GTT domain upon first access.
3261         */
3262        if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3263                mb();
3264
3265        old_write_domain = obj->base.write_domain;
3266        old_read_domains = obj->base.read_domains;
3267
3268        /* It should now be out of any other write domains, and we can update
3269         * the domain values for our changes.
3270         */
3271        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3272        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3273        if (write) {
3274                obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3275                obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3276                obj->dirty = 1;
3277        }
3278
3279        trace_i915_gem_object_change_domain(obj,
3280                                            old_read_domains,
3281                                            old_write_domain);
3282
3283        /* And bump the LRU for this access */
3284        i915_gem_object_bump_inactive_ggtt(obj);
3285
3286        return 0;
3287}
3288
3289/**
3290 * Changes the cache-level of an object across all VMA.
3291 * @obj: object to act on
3292 * @cache_level: new cache level to set for the object
3293 *
3294 * After this function returns, the object will be in the new cache-level
3295 * across all GTT and the contents of the backing storage will be coherent,
3296 * with respect to the new cache-level. In order to keep the backing storage
3297 * coherent for all users, we only allow a single cache level to be set
3298 * globally on the object and prevent it from being changed whilst the
3299 * hardware is reading from the object. That is if the object is currently
3300 * on the scanout it will be set to uncached (or equivalent display
3301 * cache coherency) and all non-MOCS GPU access will also be uncached so
3302 * that all direct access to the scanout remains coherent.
3303 */
3304int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3305                                    enum i915_cache_level cache_level)
3306{
3307        struct i915_vma *vma;
3308        int ret = 0;
3309
3310        if (obj->cache_level == cache_level)
3311                goto out;
3312
3313        /* Inspect the list of currently bound VMA and unbind any that would
3314         * be invalid given the new cache-level. This is principally to
3315         * catch the issue of the CS prefetch crossing page boundaries and
3316         * reading an invalid PTE on older architectures.
3317         */
3318restart:
3319        list_for_each_entry(vma, &obj->vma_list, obj_link) {
3320                if (!drm_mm_node_allocated(&vma->node))
3321                        continue;
3322
3323                if (i915_vma_is_pinned(vma)) {
3324                        DRM_DEBUG("can not change the cache level of pinned objects\n");
3325                        return -EBUSY;
3326                }
3327
3328                if (i915_gem_valid_gtt_space(vma, cache_level))
3329                        continue;
3330
3331                ret = i915_vma_unbind(vma);
3332                if (ret)
3333                        return ret;
3334
3335                /* As unbinding may affect other elements in the
3336                 * obj->vma_list (due to side-effects from retiring
3337                 * an active vma), play safe and restart the iterator.
3338                 */
3339                goto restart;
3340        }
3341
3342        /* We can reuse the existing drm_mm nodes but need to change the
3343         * cache-level on the PTE. We could simply unbind them all and
3344         * rebind with the correct cache-level on next use. However since
3345         * we already have a valid slot, dma mapping, pages etc, we may as
3346         * rewrite the PTE in the belief that doing so tramples upon less
3347         * state and so involves less work.
3348         */
3349        if (obj->bind_count) {
3350                /* Before we change the PTE, the GPU must not be accessing it.
3351                 * If we wait upon the object, we know that all the bound
3352                 * VMA are no longer active.
3353                 */
3354                ret = i915_gem_object_wait_rendering(obj, false);
3355                if (ret)
3356                        return ret;
3357
3358                if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) {
3359                        /* Access to snoopable pages through the GTT is
3360                         * incoherent and on some machines causes a hard
3361                         * lockup. Relinquish the CPU mmaping to force
3362                         * userspace to refault in the pages and we can
3363                         * then double check if the GTT mapping is still
3364                         * valid for that pointer access.
3365                         */
3366                        i915_gem_release_mmap(obj);
3367
3368                        /* As we no longer need a fence for GTT access,
3369                         * we can relinquish it now (and so prevent having
3370                         * to steal a fence from someone else on the next
3371                         * fence request). Note GPU activity would have
3372                         * dropped the fence as all snoopable access is
3373                         * supposed to be linear.
3374                         */
3375                        list_for_each_entry(vma, &obj->vma_list, obj_link) {
3376                                ret = i915_vma_put_fence(vma);
3377                                if (ret)
3378                                        return ret;
3379                        }
3380                } else {
3381                        /* We either have incoherent backing store and
3382                         * so no GTT access or the architecture is fully
3383                         * coherent. In such cases, existing GTT mmaps
3384                         * ignore the cache bit in the PTE and we can
3385                         * rewrite it without confusing the GPU or having
3386                         * to force userspace to fault back in its mmaps.
3387                         */
3388                }
3389
3390                list_for_each_entry(vma, &obj->vma_list, obj_link) {
3391                        if (!drm_mm_node_allocated(&vma->node))
3392                                continue;
3393
3394                        ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3395                        if (ret)
3396                                return ret;
3397                }
3398        }
3399
3400        list_for_each_entry(vma, &obj->vma_list, obj_link)
3401                vma->node.color = cache_level;
3402        obj->cache_level = cache_level;
3403
3404out:
3405        /* Flush the dirty CPU caches to the backing storage so that the
3406         * object is now coherent at its new cache level (with respect
3407         * to the access domain).
3408         */
3409        if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3410                if (i915_gem_clflush_object(obj, true))
3411                        i915_gem_chipset_flush(to_i915(obj->base.dev));
3412        }
3413
3414        return 0;
3415}
3416
3417int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3418                               struct drm_file *file)
3419{
3420        struct drm_i915_gem_caching *args = data;
3421        struct drm_i915_gem_object *obj;
3422
3423        obj = i915_gem_object_lookup(file, args->handle);
3424        if (!obj)
3425                return -ENOENT;
3426
3427        switch (obj->cache_level) {
3428        case I915_CACHE_LLC:
3429        case I915_CACHE_L3_LLC:
3430                args->caching = I915_CACHING_CACHED;
3431                break;
3432
3433        case I915_CACHE_WT:
3434                args->caching = I915_CACHING_DISPLAY;
3435                break;
3436
3437        default:
3438                args->caching = I915_CACHING_NONE;
3439                break;
3440        }
3441
3442        i915_gem_object_put_unlocked(obj);
3443        return 0;
3444}
3445
3446int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3447                               struct drm_file *file)
3448{
3449        struct drm_i915_private *dev_priv = to_i915(dev);
3450        struct drm_i915_gem_caching *args = data;
3451        struct drm_i915_gem_object *obj;
3452        enum i915_cache_level level;
3453        int ret;
3454
3455        switch (args->caching) {
3456        case I915_CACHING_NONE:
3457                level = I915_CACHE_NONE;
3458                break;
3459        case I915_CACHING_CACHED:
3460                /*
3461                 * Due to a HW issue on BXT A stepping, GPU stores via a
3462                 * snooped mapping may leave stale data in a corresponding CPU
3463                 * cacheline, whereas normally such cachelines would get
3464                 * invalidated.
3465                 */
3466                if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3467                        return -ENODEV;
3468
3469                level = I915_CACHE_LLC;
3470                break;
3471        case I915_CACHING_DISPLAY:
3472                level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3473                break;
3474        default:
3475                return -EINVAL;
3476        }
3477
3478        intel_runtime_pm_get(dev_priv);
3479
3480        ret = i915_mutex_lock_interruptible(dev);
3481        if (ret)
3482                goto rpm_put;
3483
3484        obj = i915_gem_object_lookup(file, args->handle);
3485        if (!obj) {
3486                ret = -ENOENT;
3487                goto unlock;
3488        }
3489
3490        ret = i915_gem_object_set_cache_level(obj, level);
3491
3492        i915_gem_object_put(obj);
3493unlock:
3494        mutex_unlock(&dev->struct_mutex);
3495rpm_put:
3496        intel_runtime_pm_put(dev_priv);
3497
3498        return ret;
3499}
3500
3501/*
3502 * Prepare buffer for display plane (scanout, cursors, etc).
3503 * Can be called from an uninterruptible phase (modesetting) and allows
3504 * any flushes to be pipelined (for pageflips).
3505 */
3506struct i915_vma *
3507i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3508                                     u32 alignment,
3509                                     const struct i915_ggtt_view *view)
3510{
3511        struct i915_vma *vma;
3512        u32 old_read_domains, old_write_domain;
3513        int ret;
3514
3515        /* Mark the pin_display early so that we account for the
3516         * display coherency whilst setting up the cache domains.
3517         */
3518        obj->pin_display++;
3519
3520        /* The display engine is not coherent with the LLC cache on gen6.  As
3521         * a result, we make sure that the pinning that is about to occur is
3522         * done with uncached PTEs. This is lowest common denominator for all
3523         * chipsets.
3524         *
3525         * However for gen6+, we could do better by using the GFDT bit instead
3526         * of uncaching, which would allow us to flush all the LLC-cached data
3527         * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3528         */
3529        ret = i915_gem_object_set_cache_level(obj,
3530                                              HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3531        if (ret) {
3532                vma = ERR_PTR(ret);
3533                goto err_unpin_display;
3534        }
3535
3536        /* As the user may map the buffer once pinned in the display plane
3537         * (e.g. libkms for the bootup splash), we have to ensure that we
3538         * always use map_and_fenceable for all scanout buffers. However,
3539         * it may simply be too big to fit into mappable, in which case
3540         * put it anyway and hope that userspace can cope (but always first
3541         * try to preserve the existing ABI).
3542         */
3543        vma = ERR_PTR(-ENOSPC);
3544        if (view->type == I915_GGTT_VIEW_NORMAL)
3545                vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
3546                                               PIN_MAPPABLE | PIN_NONBLOCK);
3547        if (IS_ERR(vma)) {
3548                struct drm_i915_private *i915 = to_i915(obj->base.dev);
3549                unsigned int flags;
3550
3551                /* Valleyview is definitely limited to scanning out the first
3552                 * 512MiB. Lets presume this behaviour was inherited from the
3553                 * g4x display engine and that all earlier gen are similarly
3554                 * limited. Testing suggests that it is a little more
3555                 * complicated than this. For example, Cherryview appears quite
3556                 * happy to scanout from anywhere within its global aperture.
3557                 */
3558                flags = 0;
3559                if (HAS_GMCH_DISPLAY(i915))
3560                        flags = PIN_MAPPABLE;
3561                vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
3562        }
3563        if (IS_ERR(vma))
3564                goto err_unpin_display;
3565
3566        vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
3567
3568        i915_gem_object_flush_cpu_write_domain(obj);
3569
3570        old_write_domain = obj->base.write_domain;
3571        old_read_domains = obj->base.read_domains;
3572
3573        /* It should now be out of any other write domains, and we can update
3574         * the domain values for our changes.
3575         */
3576        obj->base.write_domain = 0;
3577        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3578
3579        trace_i915_gem_object_change_domain(obj,
3580                                            old_read_domains,
3581                                            old_write_domain);
3582
3583        return vma;
3584
3585err_unpin_display:
3586        obj->pin_display--;
3587        return vma;
3588}
3589
3590void
3591i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
3592{
3593        if (WARN_ON(vma->obj->pin_display == 0))
3594                return;
3595
3596        if (--vma->obj->pin_display == 0)
3597                vma->display_alignment = 0;
3598
3599        /* Bump the LRU to try and avoid premature eviction whilst flipping  */
3600        if (!i915_vma_is_active(vma))
3601                list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3602
3603        i915_vma_unpin(vma);
3604}
3605
3606/**
3607 * Moves a single object to the CPU read, and possibly write domain.
3608 * @obj: object to act on
3609 * @write: requesting write or read-only access
3610 *
3611 * This function returns when the move is complete, including waiting on
3612 * flushes to occur.
3613 */
3614int
3615i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3616{
3617        uint32_t old_write_domain, old_read_domains;
3618        int ret;
3619
3620        ret = i915_gem_object_wait_rendering(obj, !write);
3621        if (ret)
3622                return ret;
3623
3624        if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3625                return 0;
3626
3627        i915_gem_object_flush_gtt_write_domain(obj);
3628
3629        old_write_domain = obj->base.write_domain;
3630        old_read_domains = obj->base.read_domains;
3631
3632        /* Flush the CPU cache if it's still invalid. */
3633        if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3634                i915_gem_clflush_object(obj, false);
3635
3636                obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3637        }
3638
3639        /* It should now be out of any other write domains, and we can update
3640         * the domain values for our changes.
3641         */
3642        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3643
3644        /* If we're writing through the CPU, then the GPU read domains will
3645         * need to be invalidated at next use.
3646         */
3647        if (write) {
3648                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3649                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3650        }
3651
3652        trace_i915_gem_object_change_domain(obj,
3653                                            old_read_domains,
3654                                            old_write_domain);
3655
3656        return 0;
3657}
3658
3659/* Throttle our rendering by waiting until the ring has completed our requests
3660 * emitted over 20 msec ago.
3661 *
3662 * Note that if we were to use the current jiffies each time around the loop,
3663 * we wouldn't escape the function with any frames outstanding if the time to
3664 * render a frame was over 20ms.
3665 *
3666 * This should get us reasonable parallelism between CPU and GPU but also
3667 * relatively low latency when blocking on a particular request to finish.
3668 */
3669static int
3670i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3671{
3672        struct drm_i915_private *dev_priv = to_i915(dev);
3673        struct drm_i915_file_private *file_priv = file->driver_priv;
3674        unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3675        struct drm_i915_gem_request *request, *target = NULL;
3676        int ret;
3677
3678        ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3679        if (ret)
3680                return ret;
3681
3682        /* ABI: return -EIO if already wedged */
3683        if (i915_terminally_wedged(&dev_priv->gpu_error))
3684                return -EIO;
3685
3686        spin_lock(&file_priv->mm.lock);
3687        list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3688                if (time_after_eq(request->emitted_jiffies, recent_enough))
3689                        break;
3690
3691                /*
3692                 * Note that the request might not have been submitted yet.
3693                 * In which case emitted_jiffies will be zero.
3694                 */
3695                if (!request->emitted_jiffies)
3696                        continue;
3697
3698                target = request;
3699        }
3700        if (target)
3701                i915_gem_request_get(target);
3702        spin_unlock(&file_priv->mm.lock);
3703
3704        if (target == NULL)
3705                return 0;
3706
3707        ret = i915_wait_request(target, I915_WAIT_INTERRUPTIBLE, NULL, NULL);
3708        i915_gem_request_put(target);
3709
3710        return ret;
3711}
3712
3713static bool
3714i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
3715{
3716        if (!drm_mm_node_allocated(&vma->node))
3717                return false;
3718
3719        if (vma->node.size < size)
3720                return true;
3721
3722        if (alignment && vma->node.start & (alignment - 1))
3723                return true;
3724
3725        if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
3726                return true;
3727
3728        if (flags & PIN_OFFSET_BIAS &&
3729            vma->node.start < (flags & PIN_OFFSET_MASK))
3730                return true;
3731
3732        if (flags & PIN_OFFSET_FIXED &&
3733            vma->node.start != (flags & PIN_OFFSET_MASK))
3734                return true;
3735
3736        return false;
3737}
3738
3739void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3740{
3741        struct drm_i915_gem_object *obj = vma->obj;
3742        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3743        bool mappable, fenceable;
3744        u32 fence_size, fence_alignment;
3745
3746        fence_size = i915_gem_get_ggtt_size(dev_priv,
3747                                            vma->size,
3748                                            i915_gem_object_get_tiling(obj));
3749        fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
3750                                                      vma->size,
3751                                                      i915_gem_object_get_tiling(obj),
3752                                                      true);
3753
3754        fenceable = (vma->node.size == fence_size &&
3755                     (vma->node.start & (fence_alignment - 1)) == 0);
3756
3757        mappable = (vma->node.start + fence_size <=
3758                    dev_priv->ggtt.mappable_end);
3759
3760        /*
3761         * Explicitly disable for rotated VMA since the display does not
3762         * need the fence and the VMA is not accessible to other users.
3763         */
3764        if (mappable && fenceable &&
3765            vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED)
3766                vma->flags |= I915_VMA_CAN_FENCE;
3767        else
3768                vma->flags &= ~I915_VMA_CAN_FENCE;
3769}
3770
3771int __i915_vma_do_pin(struct i915_vma *vma,
3772                      u64 size, u64 alignment, u64 flags)
3773{
3774        unsigned int bound = vma->flags;
3775        int ret;
3776
3777        GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
3778        GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
3779
3780        if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
3781                ret = -EBUSY;
3782                goto err;
3783        }
3784
3785        if ((bound & I915_VMA_BIND_MASK) == 0) {
3786                ret = i915_vma_insert(vma, size, alignment, flags);
3787                if (ret)
3788                        goto err;
3789        }
3790
3791        ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
3792        if (ret)
3793                goto err;
3794
3795        if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
3796                __i915_vma_set_map_and_fenceable(vma);
3797
3798        GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
3799        return 0;
3800
3801err:
3802        __i915_vma_unpin(vma);
3803        return ret;
3804}
3805
3806struct i915_vma *
3807i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3808                         const struct i915_ggtt_view *view,
3809                         u64 size,
3810                         u64 alignment,
3811                         u64 flags)
3812{
3813        struct i915_address_space *vm = &to_i915(obj->base.dev)->ggtt.base;
3814        struct i915_vma *vma;
3815        int ret;
3816
3817        vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view);
3818        if (IS_ERR(vma))
3819                return vma;
3820
3821        if (i915_vma_misplaced(vma, size, alignment, flags)) {
3822                if (flags & PIN_NONBLOCK &&
3823                    (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
3824                        return ERR_PTR(-ENOSPC);
3825
3826                WARN(i915_vma_is_pinned(vma),
3827                     "bo is already pinned in ggtt with incorrect alignment:"
3828                     " offset=%08x, req.alignment=%llx,"
3829                     " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
3830                     i915_ggtt_offset(vma), alignment,
3831                     !!(flags & PIN_MAPPABLE),
3832                     i915_vma_is_map_and_fenceable(vma));
3833                ret = i915_vma_unbind(vma);
3834                if (ret)
3835                        return ERR_PTR(ret);
3836        }
3837
3838        ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
3839        if (ret)
3840                return ERR_PTR(ret);
3841
3842        return vma;
3843}
3844
3845static __always_inline unsigned int __busy_read_flag(unsigned int id)
3846{
3847        /* Note that we could alias engines in the execbuf API, but
3848         * that would be very unwise as it prevents userspace from
3849         * fine control over engine selection. Ahem.
3850         *
3851         * This should be something like EXEC_MAX_ENGINE instead of
3852         * I915_NUM_ENGINES.
3853         */
3854        BUILD_BUG_ON(I915_NUM_ENGINES > 16);
3855        return 0x10000 << id;
3856}
3857
3858static __always_inline unsigned int __busy_write_id(unsigned int id)
3859{
3860        /* The uABI guarantees an active writer is also amongst the read
3861         * engines. This would be true if we accessed the activity tracking
3862         * under the lock, but as we perform the lookup of the object and
3863         * its activity locklessly we can not guarantee that the last_write
3864         * being active implies that we have set the same engine flag from
3865         * last_read - hence we always set both read and write busy for
3866         * last_write.
3867         */
3868        return id | __busy_read_flag(id);
3869}
3870
3871static __always_inline unsigned int
3872__busy_set_if_active(const struct i915_gem_active *active,
3873                     unsigned int (*flag)(unsigned int id))
3874{
3875        struct drm_i915_gem_request *request;
3876
3877        request = rcu_dereference(active->request);
3878        if (!request || i915_gem_request_completed(request))
3879                return 0;
3880
3881        /* This is racy. See __i915_gem_active_get_rcu() for an in detail
3882         * discussion of how to handle the race correctly, but for reporting
3883         * the busy state we err on the side of potentially reporting the
3884         * wrong engine as being busy (but we guarantee that the result
3885         * is at least self-consistent).
3886         *
3887         * As we use SLAB_DESTROY_BY_RCU, the request may be reallocated
3888         * whilst we are inspecting it, even under the RCU read lock as we are.
3889         * This means that there is a small window for the engine and/or the
3890         * seqno to have been overwritten. The seqno will always be in the
3891         * future compared to the intended, and so we know that if that
3892         * seqno is idle (on whatever engine) our request is idle and the
3893         * return 0 above is correct.
3894         *
3895         * The issue is that if the engine is switched, it is just as likely
3896         * to report that it is busy (but since the switch happened, we know
3897         * the request should be idle). So there is a small chance that a busy
3898         * result is actually the wrong engine.
3899         *
3900         * So why don't we care?
3901         *
3902         * For starters, the busy ioctl is a heuristic that is by definition
3903         * racy. Even with perfect serialisation in the driver, the hardware
3904         * state is constantly advancing - the state we report to the user
3905         * is stale.
3906         *
3907         * The critical information for the busy-ioctl is whether the object
3908         * is idle as userspace relies on that to detect whether its next
3909         * access will stall, or if it has missed submitting commands to
3910         * the hardware allowing the GPU to stall. We never generate a
3911         * false-positive for idleness, thus busy-ioctl is reliable at the
3912         * most fundamental level, and we maintain the guarantee that a
3913         * busy object left to itself will eventually become idle (and stay
3914         * idle!).
3915         *
3916         * We allow ourselves the leeway of potentially misreporting the busy
3917         * state because that is an optimisation heuristic that is constantly
3918         * in flux. Being quickly able to detect the busy/idle state is much
3919         * more important than accurate logging of exactly which engines were
3920         * busy.
3921         *
3922         * For accuracy in reporting the engine, we could use
3923         *
3924         *      result = 0;
3925         *      request = __i915_gem_active_get_rcu(active);
3926         *      if (request) {
3927         *              if (!i915_gem_request_completed(request))
3928         *                      result = flag(request->engine->exec_id);
3929         *              i915_gem_request_put(request);
3930         *      }
3931         *
3932         * but that still remains susceptible to both hardware and userspace
3933         * races. So we accept making the result of that race slightly worse,
3934         * given the rarity of the race and its low impact on the result.
3935         */
3936        return flag(READ_ONCE(request->engine->exec_id));
3937}
3938
3939static __always_inline unsigned int
3940busy_check_reader(const struct i915_gem_active *active)
3941{
3942        return __busy_set_if_active(active, __busy_read_flag);
3943}
3944
3945static __always_inline unsigned int
3946busy_check_writer(const struct i915_gem_active *active)
3947{
3948        return __busy_set_if_active(active, __busy_write_id);
3949}
3950
3951int
3952i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3953                    struct drm_file *file)
3954{
3955        struct drm_i915_gem_busy *args = data;
3956        struct drm_i915_gem_object *obj;
3957        unsigned long active;
3958
3959        obj = i915_gem_object_lookup(file, args->handle);
3960        if (!obj)
3961                return -ENOENT;
3962
3963        args->busy = 0;
3964        active = __I915_BO_ACTIVE(obj);
3965        if (active) {
3966                int idx;
3967
3968                /* Yes, the lookups are intentionally racy.
3969                 *
3970                 * First, we cannot simply rely on __I915_BO_ACTIVE. We have
3971                 * to regard the value as stale and as our ABI guarantees
3972                 * forward progress, we confirm the status of each active
3973                 * request with the hardware.
3974                 *
3975                 * Even though we guard the pointer lookup by RCU, that only
3976                 * guarantees that the pointer and its contents remain
3977                 * dereferencable and does *not* mean that the request we
3978                 * have is the same as the one being tracked by the object.
3979                 *
3980                 * Consider that we lookup the request just as it is being
3981                 * retired and freed. We take a local copy of the pointer,
3982                 * but before we add its engine into the busy set, the other
3983                 * thread reallocates it and assigns it to a task on another
3984                 * engine with a fresh and incomplete seqno. Guarding against
3985                 * that requires careful serialisation and reference counting,
3986                 * i.e. using __i915_gem_active_get_request_rcu(). We don't,
3987                 * instead we expect that if the result is busy, which engines
3988                 * are busy is not completely reliable - we only guarantee
3989                 * that the object was busy.
3990                 */
3991                rcu_read_lock();
3992
3993                for_each_active(active, idx)
3994                        args->busy |= busy_check_reader(&obj->last_read[idx]);
3995
3996                /* For ABI sanity, we only care that the write engine is in
3997                 * the set of read engines. This should be ensured by the
3998                 * ordering of setting last_read/last_write in
3999                 * i915_vma_move_to_active(), and then in reverse in retire.
4000                 * However, for good measure, we always report the last_write
4001                 * request as a busy read as well as being a busy write.
4002                 *
4003                 * We don't care that the set of active read/write engines
4004                 * may change during construction of the result, as it is
4005                 * equally liable to change before userspace can inspect
4006                 * the result.
4007                 */
4008                args->busy |= busy_check_writer(&obj->last_write);
4009
4010                rcu_read_unlock();
4011        }
4012
4013        i915_gem_object_put_unlocked(obj);
4014        return 0;
4015}
4016
4017int
4018i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4019                        struct drm_file *file_priv)
4020{
4021        return i915_gem_ring_throttle(dev, file_priv);
4022}
4023
4024int
4025i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4026                       struct drm_file *file_priv)
4027{
4028        struct drm_i915_private *dev_priv = to_i915(dev);
4029        struct drm_i915_gem_madvise *args = data;
4030        struct drm_i915_gem_object *obj;
4031        int ret;
4032
4033        switch (args->madv) {
4034        case I915_MADV_DONTNEED:
4035        case I915_MADV_WILLNEED:
4036            break;
4037        default:
4038            return -EINVAL;
4039        }
4040
4041        ret = i915_mutex_lock_interruptible(dev);
4042        if (ret)
4043                return ret;
4044
4045        obj = i915_gem_object_lookup(file_priv, args->handle);
4046        if (!obj) {
4047                ret = -ENOENT;
4048                goto unlock;
4049        }
4050
4051        if (obj->pages &&
4052            i915_gem_object_is_tiled(obj) &&
4053            dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4054                if (obj->madv == I915_MADV_WILLNEED)
4055                        i915_gem_object_unpin_pages(obj);
4056                if (args->madv == I915_MADV_WILLNEED)
4057                        i915_gem_object_pin_pages(obj);
4058        }
4059
4060        if (obj->madv != __I915_MADV_PURGED)
4061                obj->madv = args->madv;
4062
4063        /* if the object is no longer attached, discard its backing storage */
4064        if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4065                i915_gem_object_truncate(obj);
4066
4067        args->retained = obj->madv != __I915_MADV_PURGED;
4068
4069        i915_gem_object_put(obj);
4070unlock:
4071        mutex_unlock(&dev->struct_mutex);
4072        return ret;
4073}
4074
4075void i915_gem_object_init(struct drm_i915_gem_object *obj,
4076                          const struct drm_i915_gem_object_ops *ops)
4077{
4078        int i;
4079
4080        INIT_LIST_HEAD(&obj->global_list);
4081        for (i = 0; i < I915_NUM_ENGINES; i++)
4082                init_request_active(&obj->last_read[i],
4083                                    i915_gem_object_retire__read);
4084        init_request_active(&obj->last_write,
4085                            i915_gem_object_retire__write);
4086        INIT_LIST_HEAD(&obj->obj_exec_link);
4087        INIT_LIST_HEAD(&obj->vma_list);
4088        INIT_LIST_HEAD(&obj->batch_pool_link);
4089
4090        obj->ops = ops;
4091
4092        obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
4093        obj->madv = I915_MADV_WILLNEED;
4094
4095        i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4096}
4097
4098static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4099        .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4100        .get_pages = i915_gem_object_get_pages_gtt,
4101        .put_pages = i915_gem_object_put_pages_gtt,
4102};
4103
4104struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4105                                                  size_t size)
4106{
4107        struct drm_i915_gem_object *obj;
4108        struct address_space *mapping;
4109        gfp_t mask;
4110        int ret;
4111
4112        obj = i915_gem_object_alloc(dev);
4113        if (obj == NULL)
4114                return ERR_PTR(-ENOMEM);
4115
4116        ret = drm_gem_object_init(dev, &obj->base, size);
4117        if (ret)
4118                goto fail;
4119
4120        mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4121        if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4122                /* 965gm cannot relocate objects above 4GiB. */
4123                mask &= ~__GFP_HIGHMEM;
4124                mask |= __GFP_DMA32;
4125        }
4126
4127        mapping = obj->base.filp->f_mapping;
4128        mapping_set_gfp_mask(mapping, mask);
4129
4130        i915_gem_object_init(obj, &i915_gem_object_ops);
4131
4132        obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4133        obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4134
4135        if (HAS_LLC(dev)) {
4136                /* On some devices, we can have the GPU use the LLC (the CPU
4137                 * cache) for about a 10% performance improvement
4138                 * compared to uncached.  Graphics requests other than
4139                 * display scanout are coherent with the CPU in
4140                 * accessing this cache.  This means in this mode we
4141                 * don't need to clflush on the CPU side, and on the
4142                 * GPU side we only need to flush internal caches to
4143                 * get data visible to the CPU.
4144                 *
4145                 * However, we maintain the display planes as UC, and so
4146                 * need to rebind when first used as such.
4147                 */
4148                obj->cache_level = I915_CACHE_LLC;
4149        } else
4150                obj->cache_level = I915_CACHE_NONE;
4151
4152        trace_i915_gem_object_create(obj);
4153
4154        return obj;
4155
4156fail:
4157        i915_gem_object_free(obj);
4158
4159        return ERR_PTR(ret);
4160}
4161
4162static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4163{
4164        /* If we are the last user of the backing storage (be it shmemfs
4165         * pages or stolen etc), we know that the pages are going to be
4166         * immediately released. In this case, we can then skip copying
4167         * back the contents from the GPU.
4168         */
4169
4170        if (obj->madv != I915_MADV_WILLNEED)
4171                return false;
4172
4173        if (obj->base.filp == NULL)
4174                return true;
4175
4176        /* At first glance, this looks racy, but then again so would be
4177         * userspace racing mmap against close. However, the first external
4178         * reference to the filp can only be obtained through the
4179         * i915_gem_mmap_ioctl() which safeguards us against the user
4180         * acquiring such a reference whilst we are in the middle of
4181         * freeing the object.
4182         */
4183        return atomic_long_read(&obj->base.filp->f_count) == 1;
4184}
4185
4186void i915_gem_free_object(struct drm_gem_object *gem_obj)
4187{
4188        struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4189        struct drm_device *dev = obj->base.dev;
4190        struct drm_i915_private *dev_priv = to_i915(dev);
4191        struct i915_vma *vma, *next;
4192
4193        intel_runtime_pm_get(dev_priv);
4194
4195        trace_i915_gem_object_destroy(obj);
4196
4197        /* All file-owned VMA should have been released by this point through
4198         * i915_gem_close_object(), or earlier by i915_gem_context_close().
4199         * However, the object may also be bound into the global GTT (e.g.
4200         * older GPUs without per-process support, or for direct access through
4201         * the GTT either for the user or for scanout). Those VMA still need to
4202         * unbound now.
4203         */
4204        list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4205                GEM_BUG_ON(!i915_vma_is_ggtt(vma));
4206                GEM_BUG_ON(i915_vma_is_active(vma));
4207                vma->flags &= ~I915_VMA_PIN_MASK;
4208                i915_vma_close(vma);
4209        }
4210        GEM_BUG_ON(obj->bind_count);
4211
4212        /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4213         * before progressing. */
4214        if (obj->stolen)
4215                i915_gem_object_unpin_pages(obj);
4216
4217        WARN_ON(atomic_read(&obj->frontbuffer_bits));
4218
4219        if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4220            dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4221            i915_gem_object_is_tiled(obj))
4222                i915_gem_object_unpin_pages(obj);
4223
4224        if (WARN_ON(obj->pages_pin_count))
4225                obj->pages_pin_count = 0;
4226        if (discard_backing_storage(obj))
4227                obj->madv = I915_MADV_DONTNEED;
4228        i915_gem_object_put_pages(obj);
4229
4230        BUG_ON(obj->pages);
4231
4232        if (obj->base.import_attach)
4233                drm_prime_gem_destroy(&obj->base, NULL);
4234
4235        if (obj->ops->release)
4236                obj->ops->release(obj);
4237
4238        drm_gem_object_release(&obj->base);
4239        i915_gem_info_remove_obj(dev_priv, obj->base.size);
4240
4241        kfree(obj->bit_17);
4242        i915_gem_object_free(obj);
4243
4244        intel_runtime_pm_put(dev_priv);
4245}
4246
4247int i915_gem_suspend(struct drm_device *dev)
4248{
4249        struct drm_i915_private *dev_priv = to_i915(dev);
4250        int ret;
4251
4252        intel_suspend_gt_powersave(dev_priv);
4253
4254        mutex_lock(&dev->struct_mutex);
4255
4256        /* We have to flush all the executing contexts to main memory so
4257         * that they can saved in the hibernation image. To ensure the last
4258         * context image is coherent, we have to switch away from it. That
4259         * leaves the dev_priv->kernel_context still active when
4260         * we actually suspend, and its image in memory may not match the GPU
4261         * state. Fortunately, the kernel_context is disposable and we do
4262         * not rely on its state.
4263         */
4264        ret = i915_gem_switch_to_kernel_context(dev_priv);
4265        if (ret)
4266                goto err;
4267
4268        ret = i915_gem_wait_for_idle(dev_priv,
4269                                     I915_WAIT_INTERRUPTIBLE |
4270                                     I915_WAIT_LOCKED);
4271        if (ret)
4272                goto err;
4273
4274        i915_gem_retire_requests(dev_priv);
4275
4276        i915_gem_context_lost(dev_priv);
4277        mutex_unlock(&dev->struct_mutex);
4278
4279        cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4280        cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4281        flush_delayed_work(&dev_priv->gt.idle_work);
4282
4283        /* Assert that we sucessfully flushed all the work and
4284         * reset the GPU back to its idle, low power state.
4285         */
4286        WARN_ON(dev_priv->gt.awake);
4287
4288        return 0;
4289
4290err:
4291        mutex_unlock(&dev->struct_mutex);
4292        return ret;
4293}
4294
4295void i915_gem_resume(struct drm_device *dev)
4296{
4297        struct drm_i915_private *dev_priv = to_i915(dev);
4298
4299        mutex_lock(&dev->struct_mutex);
4300        i915_gem_restore_gtt_mappings(dev);
4301
4302        /* As we didn't flush the kernel context before suspend, we cannot
4303         * guarantee that the context image is complete. So let's just reset
4304         * it and start again.
4305         */
4306        dev_priv->gt.resume(dev_priv);
4307
4308        mutex_unlock(&dev->struct_mutex);
4309}
4310
4311void i915_gem_init_swizzling(struct drm_device *dev)
4312{
4313        struct drm_i915_private *dev_priv = to_i915(dev);
4314
4315        if (INTEL_INFO(dev)->gen < 5 ||
4316            dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4317                return;
4318
4319        I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4320                                 DISP_TILE_SURFACE_SWIZZLING);
4321
4322        if (IS_GEN5(dev))
4323                return;
4324
4325        I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4326        if (IS_GEN6(dev))
4327                I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4328        else if (IS_GEN7(dev))
4329                I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4330        else if (IS_GEN8(dev))
4331                I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4332        else
4333                BUG();
4334}
4335
4336static void init_unused_ring(struct drm_device *dev, u32 base)
4337{
4338        struct drm_i915_private *dev_priv = to_i915(dev);
4339
4340        I915_WRITE(RING_CTL(base), 0);
4341        I915_WRITE(RING_HEAD(base), 0);
4342        I915_WRITE(RING_TAIL(base), 0);
4343        I915_WRITE(RING_START(base), 0);
4344}
4345
4346static void init_unused_rings(struct drm_device *dev)
4347{
4348        if (IS_I830(dev)) {
4349                init_unused_ring(dev, PRB1_BASE);
4350                init_unused_ring(dev, SRB0_BASE);
4351                init_unused_ring(dev, SRB1_BASE);
4352                init_unused_ring(dev, SRB2_BASE);
4353                init_unused_ring(dev, SRB3_BASE);
4354        } else if (IS_GEN2(dev)) {
4355                init_unused_ring(dev, SRB0_BASE);
4356                init_unused_ring(dev, SRB1_BASE);
4357        } else if (IS_GEN3(dev)) {
4358                init_unused_ring(dev, PRB1_BASE);
4359                init_unused_ring(dev, PRB2_BASE);
4360        }
4361}
4362
4363int
4364i915_gem_init_hw(struct drm_device *dev)
4365{
4366        struct drm_i915_private *dev_priv = to_i915(dev);
4367        struct intel_engine_cs *engine;
4368        int ret;
4369
4370        /* Double layer security blanket, see i915_gem_init() */
4371        intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4372
4373        if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4374                I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4375
4376        if (IS_HASWELL(dev))
4377                I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4378                           LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4379
4380        if (HAS_PCH_NOP(dev)) {
4381                if (IS_IVYBRIDGE(dev)) {
4382                        u32 temp = I915_READ(GEN7_MSG_CTL);
4383                        temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4384                        I915_WRITE(GEN7_MSG_CTL, temp);
4385                } else if (INTEL_INFO(dev)->gen >= 7) {
4386                        u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4387                        temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4388                        I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4389                }
4390        }
4391
4392        i915_gem_init_swizzling(dev);
4393
4394        /*
4395         * At least 830 can leave some of the unused rings
4396         * "active" (ie. head != tail) after resume which
4397         * will prevent c3 entry. Makes sure all unused rings
4398         * are totally idle.
4399         */
4400        init_unused_rings(dev);
4401
4402        BUG_ON(!dev_priv->kernel_context);
4403
4404        ret = i915_ppgtt_init_hw(dev);
4405        if (ret) {
4406                DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4407                goto out;
4408        }
4409
4410        /* Need to do basic initialisation of all rings first: */
4411        for_each_engine(engine, dev_priv) {
4412                ret = engine->init_hw(engine);
4413                if (ret)
4414                        goto out;
4415        }
4416
4417        intel_mocs_init_l3cc_table(dev);
4418
4419        /* We can't enable contexts until all firmware is loaded */
4420        ret = intel_guc_setup(dev);
4421        if (ret)
4422                goto out;
4423
4424out:
4425        intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4426        return ret;
4427}
4428
4429bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4430{
4431        if (INTEL_INFO(dev_priv)->gen < 6)
4432                return false;
4433
4434        /* TODO: make semaphores and Execlists play nicely together */
4435        if (i915.enable_execlists)
4436                return false;
4437
4438        if (value >= 0)
4439                return value;
4440
4441#ifdef CONFIG_INTEL_IOMMU
4442        /* Enable semaphores on SNB when IO remapping is off */
4443        if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4444                return false;
4445#endif
4446
4447        return true;
4448}
4449
4450int i915_gem_init(struct drm_device *dev)
4451{
4452        struct drm_i915_private *dev_priv = to_i915(dev);
4453        int ret;
4454
4455        mutex_lock(&dev->struct_mutex);
4456
4457        if (!i915.enable_execlists) {
4458                dev_priv->gt.resume = intel_legacy_submission_resume;
4459                dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4460        } else {
4461                dev_priv->gt.resume = intel_lr_context_resume;
4462                dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4463        }
4464
4465        /* This is just a security blanket to placate dragons.
4466         * On some systems, we very sporadically observe that the first TLBs
4467         * used by the CS may be stale, despite us poking the TLB reset. If
4468         * we hold the forcewake during initialisation these problems
4469         * just magically go away.
4470         */
4471        intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4472
4473        i915_gem_init_userptr(dev_priv);
4474
4475        ret = i915_gem_init_ggtt(dev_priv);
4476        if (ret)
4477                goto out_unlock;
4478
4479        ret = i915_gem_context_init(dev);
4480        if (ret)
4481                goto out_unlock;
4482
4483        ret = intel_engines_init(dev);
4484        if (ret)
4485                goto out_unlock;
4486
4487        ret = i915_gem_init_hw(dev);
4488        if (ret == -EIO) {
4489                /* Allow engine initialisation to fail by marking the GPU as
4490                 * wedged. But we only want to do this where the GPU is angry,
4491                 * for all other failure, such as an allocation failure, bail.
4492                 */
4493                DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4494                i915_gem_set_wedged(dev_priv);
4495                ret = 0;
4496        }
4497
4498out_unlock:
4499        intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4500        mutex_unlock(&dev->struct_mutex);
4501
4502        return ret;
4503}
4504
4505void
4506i915_gem_cleanup_engines(struct drm_device *dev)
4507{
4508        struct drm_i915_private *dev_priv = to_i915(dev);
4509        struct intel_engine_cs *engine;
4510
4511        for_each_engine(engine, dev_priv)
4512                dev_priv->gt.cleanup_engine(engine);
4513}
4514
4515static void
4516init_engine_lists(struct intel_engine_cs *engine)
4517{
4518        INIT_LIST_HEAD(&engine->request_list);
4519}
4520
4521void
4522i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4523{
4524        struct drm_device *dev = &dev_priv->drm;
4525        int i;
4526
4527        if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4528            !IS_CHERRYVIEW(dev_priv))
4529                dev_priv->num_fence_regs = 32;
4530        else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4531                 IS_I945GM(dev_priv) || IS_G33(dev_priv))
4532                dev_priv->num_fence_regs = 16;
4533        else
4534                dev_priv->num_fence_regs = 8;
4535
4536        if (intel_vgpu_active(dev_priv))
4537                dev_priv->num_fence_regs =
4538                                I915_READ(vgtif_reg(avail_rs.fence_num));
4539
4540        /* Initialize fence registers to zero */
4541        for (i = 0; i < dev_priv->num_fence_regs; i++) {
4542                struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
4543
4544                fence->i915 = dev_priv;
4545                fence->id = i;
4546                list_add_tail(&fence->link, &dev_priv->mm.fence_list);
4547        }
4548        i915_gem_restore_fences(dev);
4549
4550        i915_gem_detect_bit_6_swizzle(dev);
4551}
4552
4553void
4554i915_gem_load_init(struct drm_device *dev)
4555{
4556        struct drm_i915_private *dev_priv = to_i915(dev);
4557        int i;
4558
4559        dev_priv->objects =
4560                kmem_cache_create("i915_gem_object",
4561                                  sizeof(struct drm_i915_gem_object), 0,
4562                                  SLAB_HWCACHE_ALIGN,
4563                                  NULL);
4564        dev_priv->vmas =
4565                kmem_cache_create("i915_gem_vma",
4566                                  sizeof(struct i915_vma), 0,
4567                                  SLAB_HWCACHE_ALIGN,
4568                                  NULL);
4569        dev_priv->requests =
4570                kmem_cache_create("i915_gem_request",
4571                                  sizeof(struct drm_i915_gem_request), 0,
4572                                  SLAB_HWCACHE_ALIGN |
4573                                  SLAB_RECLAIM_ACCOUNT |
4574                                  SLAB_DESTROY_BY_RCU,
4575                                  NULL);
4576
4577        INIT_LIST_HEAD(&dev_priv->context_list);
4578        INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4579        INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4580        INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4581        for (i = 0; i < I915_NUM_ENGINES; i++)
4582                init_engine_lists(&dev_priv->engine[i]);
4583        INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4584                          i915_gem_retire_work_handler);
4585        INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4586                          i915_gem_idle_work_handler);
4587        init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4588        init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4589
4590        dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4591
4592        init_waitqueue_head(&dev_priv->pending_flip_queue);
4593
4594        dev_priv->mm.interruptible = true;
4595
4596        atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
4597
4598        spin_lock_init(&dev_priv->fb_tracking.lock);
4599}
4600
4601void i915_gem_load_cleanup(struct drm_device *dev)
4602{
4603        struct drm_i915_private *dev_priv = to_i915(dev);
4604
4605        kmem_cache_destroy(dev_priv->requests);
4606        kmem_cache_destroy(dev_priv->vmas);
4607        kmem_cache_destroy(dev_priv->objects);
4608
4609        /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
4610        rcu_barrier();
4611}
4612
4613int i915_gem_freeze(struct drm_i915_private *dev_priv)
4614{
4615        intel_runtime_pm_get(dev_priv);
4616
4617        mutex_lock(&dev_priv->drm.struct_mutex);
4618        i915_gem_shrink_all(dev_priv);
4619        mutex_unlock(&dev_priv->drm.struct_mutex);
4620
4621        intel_runtime_pm_put(dev_priv);
4622
4623        return 0;
4624}
4625
4626int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4627{
4628        struct drm_i915_gem_object *obj;
4629        struct list_head *phases[] = {
4630                &dev_priv->mm.unbound_list,
4631                &dev_priv->mm.bound_list,
4632                NULL
4633        }, **p;
4634
4635        /* Called just before we write the hibernation image.
4636         *
4637         * We need to update the domain tracking to reflect that the CPU
4638         * will be accessing all the pages to create and restore from the
4639         * hibernation, and so upon restoration those pages will be in the
4640         * CPU domain.
4641         *
4642         * To make sure the hibernation image contains the latest state,
4643         * we update that state just before writing out the image.
4644         *
4645         * To try and reduce the hibernation image, we manually shrink
4646         * the objects as well.
4647         */
4648
4649        mutex_lock(&dev_priv->drm.struct_mutex);
4650        i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
4651
4652        for (p = phases; *p; p++) {
4653                list_for_each_entry(obj, *p, global_list) {
4654                        obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4655                        obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4656                }
4657        }
4658        mutex_unlock(&dev_priv->drm.struct_mutex);
4659
4660        return 0;
4661}
4662
4663void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4664{
4665        struct drm_i915_file_private *file_priv = file->driver_priv;
4666        struct drm_i915_gem_request *request;
4667
4668        /* Clean up our request list when the client is going away, so that
4669         * later retire_requests won't dereference our soon-to-be-gone
4670         * file_priv.
4671         */
4672        spin_lock(&file_priv->mm.lock);
4673        list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4674                request->file_priv = NULL;
4675        spin_unlock(&file_priv->mm.lock);
4676
4677        if (!list_empty(&file_priv->rps.link)) {
4678                spin_lock(&to_i915(dev)->rps.client_lock);
4679                list_del(&file_priv->rps.link);
4680                spin_unlock(&to_i915(dev)->rps.client_lock);
4681        }
4682}
4683
4684int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4685{
4686        struct drm_i915_file_private *file_priv;
4687        int ret;
4688
4689        DRM_DEBUG_DRIVER("\n");
4690
4691        file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4692        if (!file_priv)
4693                return -ENOMEM;
4694
4695        file->driver_priv = file_priv;
4696        file_priv->dev_priv = to_i915(dev);
4697        file_priv->file = file;
4698        INIT_LIST_HEAD(&file_priv->rps.link);
4699
4700        spin_lock_init(&file_priv->mm.lock);
4701        INIT_LIST_HEAD(&file_priv->mm.request_list);
4702
4703        file_priv->bsd_engine = -1;
4704
4705        ret = i915_gem_context_open(dev, file);
4706        if (ret)
4707                kfree(file_priv);
4708
4709        return ret;
4710}
4711
4712/**
4713 * i915_gem_track_fb - update frontbuffer tracking
4714 * @old: current GEM buffer for the frontbuffer slots
4715 * @new: new GEM buffer for the frontbuffer slots
4716 * @frontbuffer_bits: bitmask of frontbuffer slots
4717 *
4718 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4719 * from @old and setting them in @new. Both @old and @new can be NULL.
4720 */
4721void i915_gem_track_fb(struct drm_i915_gem_object *old,
4722                       struct drm_i915_gem_object *new,
4723                       unsigned frontbuffer_bits)
4724{
4725        /* Control of individual bits within the mask are guarded by
4726         * the owning plane->mutex, i.e. we can never see concurrent
4727         * manipulation of individual bits. But since the bitfield as a whole
4728         * is updated using RMW, we need to use atomics in order to update
4729         * the bits.
4730         */
4731        BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
4732                     sizeof(atomic_t) * BITS_PER_BYTE);
4733
4734        if (old) {
4735                WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
4736                atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
4737        }
4738
4739        if (new) {
4740                WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
4741                atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
4742        }
4743}
4744
4745/* Like i915_gem_object_get_page(), but mark the returned page dirty */
4746struct page *
4747i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
4748{
4749        struct page *page;
4750
4751        /* Only default objects have per-page dirty tracking */
4752        if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
4753                return NULL;
4754
4755        page = i915_gem_object_get_page(obj, n);
4756        set_page_dirty(page);
4757        return page;
4758}
4759
4760/* Allocate a new GEM object and fill it with the supplied data */
4761struct drm_i915_gem_object *
4762i915_gem_object_create_from_data(struct drm_device *dev,
4763                                 const void *data, size_t size)
4764{
4765        struct drm_i915_gem_object *obj;
4766        struct sg_table *sg;
4767        size_t bytes;
4768        int ret;
4769
4770        obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4771        if (IS_ERR(obj))
4772                return obj;
4773
4774        ret = i915_gem_object_set_to_cpu_domain(obj, true);
4775        if (ret)
4776                goto fail;
4777
4778        ret = i915_gem_object_get_pages(obj);
4779        if (ret)
4780                goto fail;
4781
4782        i915_gem_object_pin_pages(obj);
4783        sg = obj->pages;
4784        bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4785        obj->dirty = 1;         /* Backing store is now out of date */
4786        i915_gem_object_unpin_pages(obj);
4787
4788        if (WARN_ON(bytes != size)) {
4789                DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4790                ret = -EFAULT;
4791                goto fail;
4792        }
4793
4794        return obj;
4795
4796fail:
4797        i915_gem_object_put(obj);
4798        return ERR_PTR(ret);
4799}
Note: See TracBrowser for help on using the repository browser.