Rev 5367 | Rev 6088 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5367 | Rev 6084 | ||
---|---|---|---|
Line 1... | Line 1... | ||
1 | /* |
1 | /* |
2 | * Copyright © 2008 Intel Corporation |
2 | * Copyright © 2008-2015 Intel Corporation |
3 | * |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
Line 27... | Line 27... | ||
27 | 27 | ||
28 | #include |
28 | #include |
29 | #include |
29 | #include |
30 | #include |
30 | #include |
- | 31 | #include "i915_drv.h" |
|
31 | #include "i915_drv.h" |
32 | #include "i915_vgpu.h" |
32 | #include "i915_trace.h" |
33 | #include "i915_trace.h" |
33 | #include "intel_drv.h" |
34 | #include "intel_drv.h" |
34 | #include |
35 | #include |
35 | #include |
36 | #include |
36 | //#include |
37 | //#include |
37 | #include |
38 | #include |
- | 39 | #include |
|
Line 38... | Line 40... | ||
38 | #include |
40 | #define RQ_BUG_ON(expr) |
Line 39... | Line 41... | ||
39 | 41 | ||
40 | extern int x86_clflush_size; |
42 | extern int x86_clflush_size; |
41 | 43 | ||
Line 42... | Line -... | ||
42 | #define PROT_READ 0x1 /* page can be read */ |
- | |
43 | #define PROT_WRITE 0x2 /* page can be written */ |
- | |
44 | #define MAP_SHARED 0x01 /* Share changes */ |
- | |
45 | - | ||
46 | - | ||
47 | u64 nsecs_to_jiffies64(u64 n) |
- | |
48 | { |
- | |
49 | #if (NSEC_PER_SEC % HZ) == 0 |
- | |
50 | /* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */ |
- | |
51 | return div_u64(n, NSEC_PER_SEC / HZ); |
- | |
52 | #elif (HZ % 512) == 0 |
- | |
53 | /* overflow after 292 years if HZ = 1024 */ |
- | |
54 | return div_u64(n * HZ / 512, NSEC_PER_SEC / 512); |
- | |
55 | #else |
- | |
56 | /* |
- | |
57 | * Generic case - optimized for cases where HZ is a multiple of 3. |
- | |
58 | * overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc. |
- | |
59 | */ |
- | |
60 | return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ); |
- | |
61 | #endif |
- | |
62 | } |
- | |
63 | - | ||
Line 64... | Line 44... | ||
64 | unsigned long nsecs_to_jiffies(u64 n) |
44 | #define PROT_READ 0x1 /* page can be read */ |
Line 65... | Line 45... | ||
65 | { |
45 | #define PROT_WRITE 0x2 /* page can be written */ |
66 | return (unsigned long)nsecs_to_jiffies64(n); |
46 | #define MAP_SHARED 0x01 /* Share changes */ |
Line 78... | Line 58... | ||
78 | 58 | ||
Line 79... | Line 59... | ||
79 | #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO) |
59 | #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO) |
80 | 60 | ||
81 | - | ||
82 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
- | |
83 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, |
- | |
84 | bool force); |
- | |
85 | static __must_check int |
61 | |
86 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, |
62 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
87 | bool readonly); |
- | |
88 | static void |
- | |
89 | i915_gem_object_retire(struct drm_i915_gem_object *obj); |
63 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
90 | 64 | static void |
|
91 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
- | |
92 | struct drm_i915_gem_object *obj); |
- | |
93 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
- | |
94 | struct drm_i915_fence_reg *fence, |
- | |
95 | bool enable); |
- | |
Line 96... | Line 65... | ||
96 | 65 | i915_gem_object_retire__write(struct drm_i915_gem_object *obj); |
|
97 | 66 | static void |
|
98 | static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv); |
67 | i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring); |
99 | 68 | ||
Line 109... | Line 78... | ||
109 | return true; |
78 | return true; |
Line 110... | Line 79... | ||
110 | 79 | ||
111 | return obj->pin_display; |
80 | return obj->pin_display; |
Line 112... | Line -... | ||
112 | } |
- | |
113 | - | ||
114 | static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) |
- | |
115 | { |
- | |
116 | if (obj->tiling_mode) |
- | |
117 | i915_gem_release_mmap(obj); |
- | |
118 | - | ||
119 | /* As we do not have an associated fence register, we will force |
- | |
120 | * a tiling change if we ever need to acquire one. |
- | |
121 | */ |
- | |
122 | obj->fence_dirty = false; |
- | |
123 | obj->fence_reg = I915_FENCE_REG_NONE; |
- | |
124 | } |
81 | } |
125 | 82 | ||
126 | /* some bookkeeping */ |
83 | /* some bookkeeping */ |
127 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
84 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
128 | size_t size) |
85 | size_t size) |
Line 187... | Line 144... | ||
187 | 144 | ||
188 | WARN_ON(i915_verify_lists(dev)); |
145 | WARN_ON(i915_verify_lists(dev)); |
189 | return 0; |
146 | return 0; |
Line 190... | Line -... | ||
190 | } |
- | |
191 | - | ||
192 | static inline bool |
- | |
193 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
- | |
194 | { |
- | |
195 | return i915_gem_obj_bound_any(obj) && !obj->active; |
- | |
196 | } |
147 | } |
197 | 148 | ||
198 | int |
149 | int |
199 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, |
150 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, |
200 | struct drm_file *file) |
151 | struct drm_file *file) |
201 | { |
152 | { |
- | 153 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
202 | struct drm_i915_private *dev_priv = dev->dev_private; |
154 | struct drm_i915_gem_get_aperture *args = data; |
203 | struct drm_i915_gem_get_aperture *args = data; |
155 | struct i915_gtt *ggtt = &dev_priv->gtt; |
Line 204... | Line 156... | ||
204 | struct drm_i915_gem_object *obj; |
156 | struct i915_vma *vma; |
205 | size_t pinned; |
157 | size_t pinned; |
206 | 158 | ||
207 | pinned = 0; |
159 | pinned = 0; |
- | 160 | mutex_lock(&dev->struct_mutex); |
|
- | 161 | list_for_each_entry(vma, &ggtt->base.active_list, mm_list) |
|
- | 162 | if (vma->pin_count) |
|
208 | mutex_lock(&dev->struct_mutex); |
163 | pinned += vma->node.size; |
209 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) |
164 | list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list) |
Line 210... | Line 165... | ||
210 | if (i915_gem_obj_is_pinned(obj)) |
165 | if (vma->pin_count) |
211 | pinned += i915_gem_obj_ggtt_size(obj); |
166 | pinned += vma->node.size; |
Line 613... | Line 568... | ||
613 | user_data = to_user_ptr(args->data_ptr); |
568 | user_data = to_user_ptr(args->data_ptr); |
614 | remain = args->size; |
569 | remain = args->size; |
Line 615... | Line 570... | ||
615 | 570 | ||
Line -... | Line 571... | ||
- | 571 | offset = i915_gem_obj_ggtt_offset(obj) + args->offset; |
|
- | 572 | ||
616 | offset = i915_gem_obj_ggtt_offset(obj) + args->offset; |
573 | intel_fb_obj_invalidate(obj, ORIGIN_GTT); |
617 | 574 | ||
618 | while (remain > 0) { |
575 | while (remain > 0) { |
619 | /* Operation in this page |
576 | /* Operation in this page |
620 | * |
577 | * |
Line 635... | Line 592... | ||
635 | remain -= page_length; |
592 | remain -= page_length; |
636 | user_data += page_length; |
593 | user_data += page_length; |
637 | offset += page_length; |
594 | offset += page_length; |
638 | } |
595 | } |
Line -... | Line 596... | ||
- | 596 | ||
- | 597 | out_flush: |
|
639 | 598 | intel_fb_obj_flush(obj, false, ORIGIN_GTT); |
|
640 | out_unpin: |
599 | out_unpin: |
641 | i915_gem_object_ggtt_unpin(obj); |
600 | i915_gem_object_ggtt_unpin(obj); |
642 | out: |
601 | out: |
643 | return ret; |
602 | return ret; |
Line 740... | Line 699... | ||
740 | * right away and we therefore have to clflush anyway. */ |
699 | * right away and we therefore have to clflush anyway. */ |
741 | needs_clflush_after = cpu_write_needs_clflush(obj); |
700 | needs_clflush_after = cpu_write_needs_clflush(obj); |
742 | ret = i915_gem_object_wait_rendering(obj, false); |
701 | ret = i915_gem_object_wait_rendering(obj, false); |
743 | if (ret) |
702 | if (ret) |
744 | return ret; |
703 | return ret; |
745 | - | ||
746 | i915_gem_object_retire(obj); |
- | |
747 | } |
704 | } |
748 | /* Same trick applies to invalidate partially written cachelines read |
705 | /* Same trick applies to invalidate partially written cachelines read |
749 | * before writing. */ |
706 | * before writing. */ |
750 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) |
707 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) |
751 | needs_clflush_before = |
708 | needs_clflush_before = |
Line 753... | Line 710... | ||
753 | 710 | ||
754 | ret = i915_gem_object_get_pages(obj); |
711 | ret = i915_gem_object_get_pages(obj); |
755 | if (ret) |
712 | if (ret) |
Line -... | Line 713... | ||
- | 713 | return ret; |
|
- | 714 | ||
756 | return ret; |
715 | intel_fb_obj_invalidate(obj, ORIGIN_CPU); |
Line 757... | Line 716... | ||
757 | 716 | ||
758 | i915_gem_object_pin_pages(obj); |
717 | i915_gem_object_pin_pages(obj); |
Line 826... | Line 785... | ||
826 | * out of the cpu write domain while we've dropped the lock. |
785 | * out of the cpu write domain while we've dropped the lock. |
827 | */ |
786 | */ |
828 | if (!needs_clflush_after && |
787 | if (!needs_clflush_after && |
829 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
788 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
830 | if (i915_gem_clflush_object(obj, obj->pin_display)) |
789 | if (i915_gem_clflush_object(obj, obj->pin_display)) |
831 | i915_gem_chipset_flush(dev); |
790 | needs_clflush_after = true; |
832 | } |
791 | } |
833 | } |
792 | } |
Line 834... | Line 793... | ||
834 | 793 | ||
835 | if (needs_clflush_after) |
794 | if (needs_clflush_after) |
- | 795 | i915_gem_chipset_flush(dev); |
|
- | 796 | else |
|
Line -... | Line 797... | ||
- | 797 | obj->cache_dirty = true; |
|
836 | i915_gem_chipset_flush(dev); |
798 | |
837 | 799 | intel_fb_obj_flush(obj, false, ORIGIN_CPU); |
|
Line 838... | Line 800... | ||
838 | return ret; |
800 | return ret; |
839 | } |
801 | } |
Line 845... | Line 807... | ||
845 | */ |
807 | */ |
846 | int |
808 | int |
847 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, |
809 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, |
848 | struct drm_file *file) |
810 | struct drm_file *file) |
849 | { |
811 | { |
- | 812 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
850 | struct drm_i915_gem_pwrite *args = data; |
813 | struct drm_i915_gem_pwrite *args = data; |
851 | struct drm_i915_gem_object *obj; |
814 | struct drm_i915_gem_object *obj; |
852 | int ret; |
815 | int ret; |
Line 853... | Line 816... | ||
853 | 816 | ||
854 | if (args->size == 0) |
817 | if (args->size == 0) |
Line -... | Line 818... | ||
- | 818 | return 0; |
|
Line 855... | Line 819... | ||
855 | return 0; |
819 | |
856 | 820 | intel_runtime_pm_get(dev_priv); |
|
857 | 821 | ||
Line 858... | Line 822... | ||
858 | ret = i915_mutex_lock_interruptible(dev); |
822 | ret = i915_mutex_lock_interruptible(dev); |
859 | if (ret) |
823 | if (ret) |
860 | return ret; |
824 | goto put_rpm; |
861 | 825 | ||
Line 903... | Line 867... | ||
903 | 867 | ||
904 | out: |
868 | out: |
905 | drm_gem_object_unreference(&obj->base); |
869 | drm_gem_object_unreference(&obj->base); |
906 | unlock: |
870 | unlock: |
- | 871 | mutex_unlock(&dev->struct_mutex); |
|
- | 872 | put_rpm: |
|
- | 873 | intel_runtime_pm_put(dev_priv); |
|
907 | mutex_unlock(&dev->struct_mutex); |
874 | |
908 | return ret; |
875 | return ret; |
Line 909... | Line 876... | ||
909 | } |
876 | } |
910 | 877 | ||
Line 920... | Line 887... | ||
920 | 887 | ||
921 | /* Recovery complete, but the reset failed ... */ |
888 | /* Recovery complete, but the reset failed ... */ |
922 | if (i915_terminally_wedged(error)) |
889 | if (i915_terminally_wedged(error)) |
Line -... | Line 890... | ||
- | 890 | return -EIO; |
|
- | 891 | ||
- | 892 | /* |
|
- | 893 | * Check if GPU Reset is in progress - we need intel_ring_begin |
|
- | 894 | * to work properly to reinit the hw state while the gpu is |
|
- | 895 | * still marked as reset-in-progress. Handle this with a flag. |
|
923 | return -EIO; |
896 | */ |
924 | 897 | if (!error->reload_in_reset) |
|
Line 925... | Line 898... | ||
925 | return -EAGAIN; |
898 | return -EAGAIN; |
926 | } |
899 | } |
Line 927... | Line -... | ||
927 | - | ||
928 | return 0; |
- | |
929 | } |
- | |
930 | - | ||
931 | /* |
- | |
932 | * Compare seqno against outstanding lazy request. Emit a request if they are |
- | |
933 | * equal. |
- | |
934 | */ |
- | |
935 | int |
- | |
936 | i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno) |
- | |
937 | { |
- | |
938 | int ret; |
- | |
939 | - | ||
940 | BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); |
- | |
941 | - | ||
942 | ret = 0; |
- | |
943 | if (seqno == ring->outstanding_lazy_seqno) |
- | |
944 | ret = i915_add_request(ring, NULL); |
- | |
945 | 900 | ||
946 | return ret; |
901 | return 0; |
947 | } |
902 | } |
948 | 903 | ||
Line 955... | Line 910... | ||
955 | struct intel_engine_cs *ring) |
910 | struct intel_engine_cs *ring) |
956 | { |
911 | { |
957 | return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings); |
912 | return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings); |
958 | } |
913 | } |
Line 959... | Line 914... | ||
959 | 914 | ||
960 | static bool can_wait_boost(struct drm_i915_file_private *file_priv) |
915 | static unsigned long local_clock_us(unsigned *cpu) |
- | 916 | { |
|
- | 917 | unsigned long t; |
|
- | 918 | ||
- | 919 | /* Cheaply and approximately convert from nanoseconds to microseconds. |
|
- | 920 | * The result and subsequent calculations are also defined in the same |
|
- | 921 | * approximate microseconds units. The principal source of timing |
|
- | 922 | * error here is from the simple truncation. |
|
- | 923 | * |
|
- | 924 | * Note that local_clock() is only defined wrt to the current CPU; |
|
- | 925 | * the comparisons are no longer valid if we switch CPUs. Instead of |
|
- | 926 | * blocking preemption for the entire busywait, we can detect the CPU |
|
- | 927 | * switch and use that as indicator of system load and a reason to |
|
- | 928 | * stop busywaiting, see busywait_stop(). |
|
961 | { |
929 | */ |
- | 930 | t = GetClockNs() >> 10; |
|
- | 931 | ||
- | 932 | return t; |
|
- | 933 | } |
|
- | 934 | ||
- | 935 | static bool busywait_stop(unsigned long timeout, unsigned cpu) |
|
- | 936 | { |
|
- | 937 | unsigned this_cpu = 0; |
|
- | 938 | ||
962 | if (file_priv == NULL) |
939 | if (time_after(local_clock_us(&this_cpu), timeout)) |
Line -... | Line 940... | ||
- | 940 | return true; |
|
- | 941 | ||
- | 942 | return this_cpu != cpu; |
|
- | 943 | } |
|
- | 944 | ||
- | 945 | static int __i915_spin_request(struct drm_i915_gem_request *req, int state) |
|
- | 946 | { |
|
- | 947 | unsigned long timeout; |
|
- | 948 | unsigned cpu; |
|
- | 949 | ||
- | 950 | /* When waiting for high frequency requests, e.g. during synchronous |
|
- | 951 | * rendering split between the CPU and GPU, the finite amount of time |
|
- | 952 | * required to set up the irq and wait upon it limits the response |
|
- | 953 | * rate. By busywaiting on the request completion for a short while we |
|
- | 954 | * can service the high frequency waits as quick as possible. However, |
|
- | 955 | * if it is a slow request, we want to sleep as quickly as possible. |
|
- | 956 | * The tradeoff between waiting and sleeping is roughly the time it |
|
- | 957 | * takes to sleep on a request, on the order of a microsecond. |
|
- | 958 | */ |
|
- | 959 | ||
- | 960 | if (req->ring->irq_refcount) |
|
- | 961 | return -EBUSY; |
|
963 | return true; |
962 | |
- | 963 | /* Only spin if we know the GPU is processing this request */ |
|
- | 964 | if (!i915_gem_request_started(req, true)) |
|
- | 965 | return -EAGAIN; |
|
- | 966 | ||
- | 967 | timeout = local_clock_us(&cpu) + 5; |
|
- | 968 | while (1 /*!need_resched()*/) { |
|
- | 969 | if (i915_gem_request_completed(req, true)) |
|
- | 970 | return 0; |
|
- | 971 | ||
- | 972 | if (busywait_stop(timeout, cpu)) |
|
- | 973 | break; |
|
- | 974 | ||
- | 975 | cpu_relax_lowlatency(); |
|
- | 976 | } |
|
- | 977 | ||
- | 978 | if (i915_gem_request_completed(req, false)) |
|
- | 979 | return 0; |
|
964 | 980 | ||
Line 965... | Line 981... | ||
965 | return !atomic_xchg(&file_priv->rps_wait_boost, true); |
981 | return -EAGAIN; |
966 | } |
982 | } |
967 | - | ||
968 | /** |
983 | |
969 | * __i915_wait_seqno - wait until execution of seqno has finished |
984 | /** |
970 | * @ring: the ring expected to report seqno |
985 | * __i915_wait_request - wait until execution of request has finished |
971 | * @seqno: duh! |
986 | * @req: duh! |
972 | * @reset_counter: reset sequence associated with the given seqno |
987 | * @reset_counter: reset sequence associated with the given request |
973 | * @interruptible: do an interruptible wait (normally yes) |
988 | * @interruptible: do an interruptible wait (normally yes) |
974 | * @timeout: in - how long to wait (NULL forever); out - how much time remaining |
989 | * @timeout: in - how long to wait (NULL forever); out - how much time remaining |
975 | * |
990 | * |
976 | * Note: It is of utmost importance that the passed in seqno and reset_counter |
991 | * Note: It is of utmost importance that the passed in seqno and reset_counter |
977 | * values have been read by the caller in an smp safe manner. Where read-side |
992 | * values have been read by the caller in an smp safe manner. Where read-side |
978 | * locks are involved, it is sufficient to read the reset_counter before |
993 | * locks are involved, it is sufficient to read the reset_counter before |
979 | * unlocking the lock that protects the seqno. For lockless tricks, the |
994 | * unlocking the lock that protects the seqno. For lockless tricks, the |
980 | * reset_counter _must_ be read before, and an appropriate smp_rmb must be |
995 | * reset_counter _must_ be read before, and an appropriate smp_rmb must be |
981 | * inserted. |
996 | * inserted. |
982 | * |
997 | * |
983 | * Returns 0 if the seqno was found within the alloted time. Else returns the |
998 | * Returns 0 if the request was found within the alloted time. Else returns the |
984 | * errno with remaining time filled in timeout argument. |
999 | * errno with remaining time filled in timeout argument. |
985 | */ |
1000 | */ |
986 | int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno, |
1001 | int __i915_wait_request(struct drm_i915_gem_request *req, |
987 | unsigned reset_counter, |
1002 | unsigned reset_counter, |
988 | bool interruptible, |
1003 | bool interruptible, |
- | 1004 | s64 *timeout, |
|
989 | s64 *timeout, |
1005 | struct intel_rps_client *rps) |
990 | struct drm_i915_file_private *file_priv) |
1006 | { |
991 | { |
1007 | struct intel_engine_cs *ring = i915_gem_request_get_ring(req); |
992 | struct drm_device *dev = ring->dev; |
1008 | struct drm_device *dev = ring->dev; |
- | 1009 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 1010 | const bool irq_test_in_progress = |
|
993 | struct drm_i915_private *dev_priv = dev->dev_private; |
1011 | ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring); |
994 | const bool irq_test_in_progress = |
1012 | int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; |
Line 995... | Line 1013... | ||
995 | ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring); |
1013 | DEFINE_WAIT(wait); |
996 | unsigned long timeout_expire; |
1014 | unsigned long timeout_expire; |
Line 997... | Line 1015... | ||
997 | s64 before, now; |
1015 | s64 before, now; |
Line 998... | Line 1016... | ||
998 | 1016 | ||
999 | wait_queue_t __wait; |
1017 | wait_queue_t __wait; |
Line 1000... | Line 1018... | ||
1000 | int ret; |
1018 | int ret; |
1001 | 1019 | ||
Line 1002... | Line -... | ||
1002 | WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled"); |
- | |
1003 | 1020 | WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled"); |
|
1004 | if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) |
1021 | |
1005 | return 0; |
1022 | if (list_empty(&req->list)) |
1006 | 1023 | return 0; |
|
- | 1024 | ||
1007 | timeout_expire = timeout ? |
1025 | if (i915_gem_request_completed(req, true)) |
- | 1026 | return 0; |
|
- | 1027 | ||
- | 1028 | timeout_expire = 0; |
|
1008 | jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0; |
1029 | if (timeout) { |
Line -... | Line 1030... | ||
- | 1030 | if (WARN_ON(*timeout < 0)) |
|
- | 1031 | return -EINVAL; |
|
- | 1032 | ||
- | 1033 | if (*timeout == 0) |
|
- | 1034 | return -ETIME; |
|
- | 1035 | ||
- | 1036 | timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout); |
|
- | 1037 | } |
|
- | 1038 | ||
- | 1039 | if (INTEL_INFO(dev_priv)->gen >= 6) |
|
- | 1040 | gen6_rps_boost(dev_priv, rps, req->emitted_jiffies); |
|
- | 1041 | ||
1009 | 1042 | /* Record current time in case interrupted by signal, or wedged */ |
|
1010 | if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) { |
1043 | trace_i915_gem_request_wait_begin(req); |
- | 1044 | before = ktime_get_raw_ns(); |
|
- | 1045 | ||
Line 1011... | Line 1046... | ||
1011 | gen6_rps_boost(dev_priv); |
1046 | /* Optimistic spin for the next jiffie before touching IRQs */ |
1012 | if (file_priv) |
1047 | ret = __i915_spin_request(req, state); |
Line 1013... | Line -... | ||
1013 | mod_delayed_work(dev_priv->wq, |
- | |
1014 | &file_priv->mm.idle_work, |
- | |
Line 1015... | Line 1048... | ||
1015 | msecs_to_jiffies(100)); |
1048 | if (ret == 0) |
1016 | } |
1049 | goto out; |
Line 1017... | Line 1050... | ||
1017 | 1050 | ||
Line 1036... | Line 1069... | ||
1036 | if (ret == 0) |
1069 | if (ret == 0) |
1037 | ret = -EAGAIN; |
1070 | ret = -EAGAIN; |
1038 | break; |
1071 | break; |
1039 | } |
1072 | } |
Line 1040... | Line 1073... | ||
1040 | 1073 | ||
1041 | if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) { |
1074 | if (i915_gem_request_completed(req, false)) { |
1042 | ret = 0; |
1075 | ret = 0; |
1043 | break; |
1076 | break; |
Line 1044... | Line 1077... | ||
1044 | } |
1077 | } |
Line 1059... | Line 1092... | ||
1059 | spin_lock_irqsave(&ring->irq_queue.lock, flags); |
1092 | spin_lock_irqsave(&ring->irq_queue.lock, flags); |
1060 | list_del_init(&__wait.task_list); |
1093 | list_del_init(&__wait.task_list); |
1061 | spin_unlock_irqrestore(&ring->irq_queue.lock, flags); |
1094 | spin_unlock_irqrestore(&ring->irq_queue.lock, flags); |
1062 | } |
1095 | } |
1063 | }; |
1096 | }; |
1064 | trace_i915_gem_request_wait_end(ring, seqno); |
- | |
Line 1065... | Line 1097... | ||
1065 | 1097 | ||
Line 1066... | Line 1098... | ||
1066 | DestroyEvent(__wait.evnt); |
1098 | DestroyEvent(__wait.evnt); |
1067 | 1099 | ||
Line 1068... | Line 1100... | ||
1068 | if (!irq_test_in_progress) |
1100 | if (!irq_test_in_progress) |
- | 1101 | ring->irq_put(ring); |
|
- | 1102 | ||
- | 1103 | // finish_wait(&ring->irq_queue, &wait); |
|
- | 1104 | out: |
|
- | 1105 | now = ktime_get_raw_ns(); |
|
- | 1106 | trace_i915_gem_request_wait_end(req); |
|
- | 1107 | ||
- | 1108 | if (timeout) { |
|
- | 1109 | s64 tres = *timeout - (now - before); |
|
- | 1110 | ||
- | 1111 | *timeout = tres < 0 ? 0 : tres; |
|
- | 1112 | ||
- | 1113 | /* |
|
- | 1114 | * Apparently ktime isn't accurate enough and occasionally has a |
|
- | 1115 | * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch |
|
- | 1116 | * things up to make the test happy. We allow up to 1 jiffy. |
|
- | 1117 | * |
|
- | 1118 | * This is a regrssion from the timespec->ktime conversion. |
|
- | 1119 | */ |
|
- | 1120 | if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000) |
|
1069 | ring->irq_put(ring); |
1121 | *timeout = 0; |
1070 | 1122 | } |
|
Line -... | Line 1123... | ||
- | 1123 | ||
- | 1124 | return ret; |
|
- | 1125 | } |
|
- | 1126 | ||
- | 1127 | int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, |
|
- | 1128 | struct drm_file *file) |
|
- | 1129 | { |
|
- | 1130 | struct drm_i915_private *dev_private; |
|
- | 1131 | struct drm_i915_file_private *file_priv; |
|
- | 1132 | ||
- | 1133 | WARN_ON(!req || !file || req->file_priv); |
|
- | 1134 | ||
- | 1135 | if (!req || !file) |
|
- | 1136 | return -EINVAL; |
|
- | 1137 | ||
- | 1138 | if (req->file_priv) |
|
- | 1139 | return -EINVAL; |
|
- | 1140 | ||
- | 1141 | dev_private = req->ring->dev->dev_private; |
|
- | 1142 | file_priv = file->driver_priv; |
|
- | 1143 | ||
- | 1144 | spin_lock(&file_priv->mm.lock); |
|
- | 1145 | req->file_priv = file_priv; |
|
- | 1146 | list_add_tail(&req->client_list, &file_priv->mm.request_list); |
|
- | 1147 | spin_unlock(&file_priv->mm.lock); |
|
- | 1148 | ||
- | 1149 | req->pid = 1; |
|
- | 1150 | ||
- | 1151 | return 0; |
|
- | 1152 | } |
|
- | 1153 | ||
- | 1154 | static inline void |
|
- | 1155 | i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) |
|
- | 1156 | { |
|
- | 1157 | struct drm_i915_file_private *file_priv = request->file_priv; |
|
- | 1158 | ||
- | 1159 | if (!file_priv) |
|
- | 1160 | return; |
|
- | 1161 | ||
- | 1162 | spin_lock(&file_priv->mm.lock); |
|
- | 1163 | list_del(&request->client_list); |
|
- | 1164 | request->file_priv = NULL; |
|
- | 1165 | spin_unlock(&file_priv->mm.lock); |
|
- | 1166 | } |
|
- | 1167 | ||
- | 1168 | static void i915_gem_request_retire(struct drm_i915_gem_request *request) |
|
- | 1169 | { |
|
- | 1170 | trace_i915_gem_request_retire(request); |
|
- | 1171 | ||
- | 1172 | /* We know the GPU must have read the request to have |
|
- | 1173 | * sent us the seqno + interrupt, so use the position |
|
- | 1174 | * of tail of the request to update the last known position |
|
- | 1175 | * of the GPU head. |
|
- | 1176 | * |
|
- | 1177 | * Note this requires that we are always called in request |
|
- | 1178 | * completion order. |
|
- | 1179 | */ |
|
- | 1180 | request->ringbuf->last_retired_head = request->postfix; |
|
- | 1181 | ||
- | 1182 | list_del_init(&request->list); |
|
- | 1183 | i915_gem_request_remove_from_client(request); |
|
- | 1184 | ||
- | 1185 | i915_gem_request_unreference(request); |
|
- | 1186 | } |
|
- | 1187 | ||
- | 1188 | static void |
|
- | 1189 | __i915_gem_request_retire__upto(struct drm_i915_gem_request *req) |
|
- | 1190 | { |
|
- | 1191 | struct intel_engine_cs *engine = req->ring; |
|
- | 1192 | struct drm_i915_gem_request *tmp; |
|
- | 1193 | ||
- | 1194 | ||
- | 1195 | if (list_empty(&req->list)) |
|
- | 1196 | return; |
|
- | 1197 | ||
- | 1198 | do { |
|
- | 1199 | tmp = list_first_entry(&engine->request_list, |
|
- | 1200 | typeof(*tmp), list); |
|
- | 1201 | ||
- | 1202 | i915_gem_request_retire(tmp); |
|
- | 1203 | } while (tmp != req); |
|
1071 | // finish_wait(&ring->irq_queue, &wait); |
1204 | |
1072 | return ret; |
1205 | WARN_ON(i915_verify_lists(engine->dev)); |
1073 | } |
1206 | } |
1074 | 1207 | ||
1075 | /** |
1208 | /** |
1076 | * Waits for a sequence number to be signaled, and cleans up the |
1209 | * Waits for a request to be signaled, and cleans up the |
1077 | * request and object lists appropriately for that event. |
1210 | * request and object lists appropriately for that event. |
1078 | */ |
1211 | */ |
1079 | int |
1212 | int |
1080 | i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno) |
- | |
1081 | { |
1213 | i915_wait_request(struct drm_i915_gem_request *req) |
1082 | struct drm_device *dev = ring->dev; |
1214 | { |
Line -... | Line 1215... | ||
- | 1215 | struct drm_device *dev; |
|
- | 1216 | struct drm_i915_private *dev_priv; |
|
- | 1217 | bool interruptible; |
|
- | 1218 | int ret; |
|
- | 1219 | ||
- | 1220 | BUG_ON(req == NULL); |
|
1083 | struct drm_i915_private *dev_priv = dev->dev_private; |
1221 | |
1084 | bool interruptible = dev_priv->mm.interruptible; |
- | |
Line 1085... | Line 1222... | ||
1085 | unsigned reset_counter; |
1222 | dev = req->ring->dev; |
1086 | int ret; |
1223 | dev_priv = dev->dev_private; |
1087 | 1224 | interruptible = dev_priv->mm.interruptible; |
|
Line 1088... | Line 1225... | ||
1088 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
1225 | |
- | 1226 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
|
- | 1227 | ||
1089 | BUG_ON(seqno == 0); |
1228 | ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); |
1090 | 1229 | if (ret) |
|
Line 1091... | Line -... | ||
1091 | ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); |
- | |
1092 | if (ret) |
- | |
1093 | return ret; |
- | |
1094 | - | ||
1095 | ret = i915_gem_check_olr(ring, seqno); |
- | |
1096 | if (ret) |
- | |
1097 | return ret; |
- | |
1098 | - | ||
1099 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
- | |
1100 | return __i915_wait_seqno(ring, seqno, reset_counter, interruptible, |
- | |
1101 | NULL, NULL); |
- | |
1102 | } |
- | |
1103 | - | ||
1104 | static int |
- | |
1105 | i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj) |
- | |
1106 | { |
- | |
1107 | if (!obj->active) |
1230 | return ret; |
1108 | return 0; |
- | |
1109 | - | ||
1110 | /* Manually manage the write flush as we may have not yet |
- | |
1111 | * retired the buffer. |
1231 | |
1112 | * |
1232 | ret = __i915_wait_request(req, |
Line 1113... | Line 1233... | ||
1113 | * Note that the last_write_seqno is always the earlier of |
1233 | atomic_read(&dev_priv->gpu_error.reset_counter), |
1114 | * the two (read/write) seqno, so if we haved successfully waited, |
1234 | interruptible, NULL, NULL); |
1115 | * we know we have passed the last write. |
1235 | if (ret) |
1116 | */ |
1236 | return ret; |
1117 | obj->last_write_seqno = 0; |
1237 | |
1118 | 1238 | __i915_gem_request_retire__upto(req); |
|
1119 | return 0; |
1239 | return 0; |
1120 | } |
1240 | } |
1121 | - | ||
1122 | /** |
- | |
1123 | * Ensures that all rendering to the object has completed and the object is |
1241 | |
Line 1124... | Line -... | ||
1124 | * safe to unbind from the GTT or access from the CPU. |
- | |
1125 | */ |
1242 | /** |
1126 | static __must_check int |
1243 | * Ensures that all rendering to the object has completed and the object is |
Line -... | Line 1244... | ||
- | 1244 | * safe to unbind from the GTT or access from the CPU. |
|
- | 1245 | */ |
|
1127 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, |
1246 | int |
- | 1247 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, |
|
- | 1248 | bool readonly) |
|
- | 1249 | { |
|
- | 1250 | int ret, i; |
|
- | 1251 | ||
- | 1252 | if (!obj->active) |
|
- | 1253 | return 0; |
|
- | 1254 | ||
- | 1255 | if (readonly) { |
|
- | 1256 | if (obj->last_write_req != NULL) { |
|
- | 1257 | ret = i915_wait_request(obj->last_write_req); |
|
- | 1258 | if (ret) |
|
- | 1259 | return ret; |
|
- | 1260 | ||
- | 1261 | i = obj->last_write_req->ring->id; |
|
1128 | bool readonly) |
1262 | if (obj->last_read_req[i] == obj->last_write_req) |
1129 | { |
1263 | i915_gem_object_retire__read(obj, i); |
Line -... | Line 1264... | ||
- | 1264 | else |
|
- | 1265 | i915_gem_object_retire__write(obj); |
|
- | 1266 | } |
|
- | 1267 | } else { |
|
- | 1268 | for (i = 0; i < I915_NUM_RINGS; i++) { |
|
- | 1269 | if (obj->last_read_req[i] == NULL) |
|
- | 1270 | continue; |
|
- | 1271 | ||
- | 1272 | ret = i915_wait_request(obj->last_read_req[i]); |
|
- | 1273 | if (ret) |
|
- | 1274 | return ret; |
|
- | 1275 | ||
- | 1276 | i915_gem_object_retire__read(obj, i); |
|
- | 1277 | } |
|
- | 1278 | RQ_BUG_ON(obj->active); |
|
- | 1279 | } |
|
- | 1280 | ||
1130 | struct intel_engine_cs *ring = obj->ring; |
1281 | return 0; |
- | 1282 | } |
|
- | 1283 | ||
1131 | u32 seqno; |
1284 | static void |
Line 1132... | Line 1285... | ||
1132 | int ret; |
1285 | i915_gem_object_retire_request(struct drm_i915_gem_object *obj, |
1133 | 1286 | struct drm_i915_gem_request *req) |
|
1134 | seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; |
1287 | { |
1135 | if (seqno == 0) |
1288 | int ring = req->ring->id; |
1136 | return 0; |
1289 | |
1137 | 1290 | if (obj->last_read_req[ring] == req) |
|
1138 | ret = i915_wait_seqno(ring, seqno); |
1291 | i915_gem_object_retire__read(obj, ring); |
1139 | if (ret) |
1292 | else if (obj->last_write_req == req) |
1140 | return ret; |
1293 | i915_gem_object_retire__write(obj); |
1141 | 1294 | ||
1142 | return i915_gem_object_wait_rendering__tail(obj); |
1295 | __i915_gem_request_retire__upto(req); |
1143 | } |
1296 | } |
1144 | - | ||
1145 | /* A nonblocking variant of the above wait. This is a highly dangerous routine |
1297 | |
Line 1146... | Line 1298... | ||
1146 | * as the object state may change during this call. |
1298 | /* A nonblocking variant of the above wait. This is a highly dangerous routine |
1147 | */ |
1299 | * as the object state may change during this call. |
Line 1148... | Line -... | ||
1148 | static __must_check int |
- | |
1149 | i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, |
1300 | */ |
1150 | struct drm_i915_file_private *file_priv, |
1301 | static __must_check int |
Line 1151... | Line 1302... | ||
1151 | bool readonly) |
1302 | i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, |
1152 | { |
1303 | struct intel_rps_client *rps, |
1153 | struct drm_device *dev = obj->base.dev; |
1304 | bool readonly) |
Line 1154... | Line -... | ||
1154 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
1155 | struct intel_engine_cs *ring = obj->ring; |
- | |
1156 | unsigned reset_counter; |
- | |
1157 | u32 seqno; |
- | |
1158 | int ret; |
1305 | { |
- | 1306 | struct drm_device *dev = obj->base.dev; |
|
- | 1307 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 1308 | struct drm_i915_gem_request *requests[I915_NUM_RINGS]; |
|
- | 1309 | unsigned reset_counter; |
|
- | 1310 | int ret, i, n = 0; |
|
- | 1311 | ||
- | 1312 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
|
- | 1313 | BUG_ON(!dev_priv->mm.interruptible); |
|
- | 1314 | ||
- | 1315 | if (!obj->active) |
|
- | 1316 | return 0; |
|
- | 1317 | ||
- | 1318 | ret = i915_gem_check_wedge(&dev_priv->gpu_error, true); |
|
- | 1319 | if (ret) |
|
- | 1320 | return ret; |
|
- | 1321 | ||
- | 1322 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
|
- | 1323 | ||
- | 1324 | if (readonly) { |
|
- | 1325 | struct drm_i915_gem_request *req; |
|
- | 1326 | ||
1159 | 1327 | req = obj->last_write_req; |
|
- | 1328 | if (req == NULL) |
|
1160 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
1329 | return 0; |
1161 | BUG_ON(!dev_priv->mm.interruptible); |
1330 | |
1162 | 1331 | requests[n++] = i915_gem_request_reference(req); |
|
- | 1332 | } else { |
|
- | 1333 | for (i = 0; i < I915_NUM_RINGS; i++) { |
|
1163 | seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; |
1334 | struct drm_i915_gem_request *req; |
- | 1335 | ||
- | 1336 | req = obj->last_read_req[i]; |
|
- | 1337 | if (req == NULL) |
|
- | 1338 | continue; |
|
1164 | if (seqno == 0) |
1339 | |
- | 1340 | requests[n++] = i915_gem_request_reference(req); |
|
Line -... | Line 1341... | ||
- | 1341 | } |
|
- | 1342 | } |
|
1165 | return 0; |
1343 | |
- | 1344 | mutex_unlock(&dev->struct_mutex); |
|
1166 | 1345 | for (i = 0; ret == 0 && i < n; i++) |
|
Line 1167... | Line 1346... | ||
1167 | ret = i915_gem_check_wedge(&dev_priv->gpu_error, true); |
1346 | ret = __i915_wait_request(requests[i], reset_counter, true, |
1168 | if (ret) |
1347 | NULL, rps); |
1169 | return ret; |
1348 | mutex_lock(&dev->struct_mutex); |
Line 1223... | Line 1402... | ||
1223 | /* Try to flush the object off the GPU without holding the lock. |
1402 | /* Try to flush the object off the GPU without holding the lock. |
1224 | * We will repeat the flush holding the lock in the normal manner |
1403 | * We will repeat the flush holding the lock in the normal manner |
1225 | * to catch cases where we are gazumped. |
1404 | * to catch cases where we are gazumped. |
1226 | */ |
1405 | */ |
1227 | ret = i915_gem_object_wait_rendering__nonblocking(obj, |
1406 | ret = i915_gem_object_wait_rendering__nonblocking(obj, |
1228 | file->driver_priv, |
1407 | to_rps_client(file), |
1229 | !write_domain); |
1408 | !write_domain); |
1230 | if (ret) |
1409 | if (ret) |
1231 | goto unref; |
1410 | goto unref; |
Line 1232... | Line 1411... | ||
1232 | 1411 | ||
1233 | if (read_domains & I915_GEM_DOMAIN_GTT) { |
1412 | if (read_domains & I915_GEM_DOMAIN_GTT) |
1234 | ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); |
- | |
1235 | - | ||
1236 | /* Silently promote "you're not bound, there was nothing to do" |
- | |
1237 | * to success, since the client was just asking us to |
- | |
1238 | * make sure everything was done. |
- | |
1239 | */ |
- | |
1240 | if (ret == -EINVAL) |
- | |
1241 | ret = 0; |
1413 | ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); |
1242 | } else { |
1414 | else |
1243 | ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); |
1415 | ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); |
- | 1416 | ||
- | 1417 | if (write_domain != 0) |
|
- | 1418 | intel_fb_obj_invalidate(obj, |
|
- | 1419 | write_domain == I915_GEM_DOMAIN_GTT ? |
|
Line 1244... | Line 1420... | ||
1244 | } |
1420 | ORIGIN_GTT : ORIGIN_CPU); |
1245 | 1421 | ||
1246 | unref: |
1422 | unref: |
1247 | drm_gem_object_unreference(&obj->base); |
1423 | drm_gem_object_unreference(&obj->base); |
Line 1271... | Line 1447... | ||
1271 | goto unlock; |
1447 | goto unlock; |
1272 | } |
1448 | } |
Line 1273... | Line 1449... | ||
1273 | 1449 | ||
1274 | /* Pinned buffers may be scanout, so flush the cache */ |
1450 | /* Pinned buffers may be scanout, so flush the cache */ |
1275 | if (obj->pin_display) |
1451 | if (obj->pin_display) |
Line 1276... | Line 1452... | ||
1276 | i915_gem_object_flush_cpu_write_domain(obj, true); |
1452 | i915_gem_object_flush_cpu_write_domain(obj); |
1277 | 1453 | ||
1278 | drm_gem_object_unreference(&obj->base); |
1454 | drm_gem_object_unreference(&obj->base); |
1279 | unlock: |
1455 | unlock: |
Line 1304... | Line 1480... | ||
1304 | { |
1480 | { |
1305 | struct drm_i915_gem_mmap *args = data; |
1481 | struct drm_i915_gem_mmap *args = data; |
1306 | struct drm_gem_object *obj; |
1482 | struct drm_gem_object *obj; |
1307 | unsigned long addr; |
1483 | unsigned long addr; |
Line -... | Line 1484... | ||
- | 1484 | ||
- | 1485 | // if (args->flags & ~(I915_MMAP_WC)) |
|
1308 | 1486 | // return -EINVAL; |
|
1309 | obj = drm_gem_object_lookup(dev, file, args->handle); |
1487 | obj = drm_gem_object_lookup(dev, file, args->handle); |
1310 | if (obj == NULL) |
1488 | if (obj == NULL) |
Line 1311... | Line 1489... | ||
1311 | return -ENOENT; |
1489 | return -ENOENT; |
Line 1364... | Line 1542... | ||
1364 | 1542 | ||
1365 | // drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping); |
1543 | // drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping); |
1366 | obj->fault_mappable = false; |
1544 | obj->fault_mappable = false; |
Line -... | Line 1545... | ||
- | 1545 | } |
|
- | 1546 | ||
- | 1547 | void |
|
- | 1548 | i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) |
|
- | 1549 | { |
|
- | 1550 | struct drm_i915_gem_object *obj; |
|
- | 1551 | ||
- | 1552 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) |
|
- | 1553 | i915_gem_release_mmap(obj); |
|
1367 | } |
1554 | } |
1368 | 1555 | ||
1369 | uint32_t |
1556 | uint32_t |
1370 | i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) |
1557 | i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) |
Line 1435... | Line 1622... | ||
1435 | if (&obj->base == NULL) { |
1622 | if (&obj->base == NULL) { |
1436 | ret = -ENOENT; |
1623 | ret = -ENOENT; |
1437 | goto unlock; |
1624 | goto unlock; |
1438 | } |
1625 | } |
Line 1439... | Line -... | ||
1439 | - | ||
1440 | if (obj->base.size > dev_priv->gtt.mappable_end) { |
- | |
1441 | ret = -E2BIG; |
- | |
1442 | goto out; |
- | |
1443 | } |
- | |
1444 | 1626 | ||
1445 | if (obj->madv != I915_MADV_WILLNEED) { |
1627 | if (obj->madv != I915_MADV_WILLNEED) { |
1446 | DRM_DEBUG("Attempting to mmap a purgeable buffer\n"); |
1628 | DRM_DEBUG("Attempting to mmap a purgeable buffer\n"); |
1447 | ret = -EFAULT; |
1629 | ret = -EFAULT; |
1448 | goto out; |
1630 | goto out; |
Line 1511... | Line 1693... | ||
1511 | struct drm_i915_gem_mmap_gtt *args = data; |
1693 | struct drm_i915_gem_mmap_gtt *args = data; |
Line 1512... | Line 1694... | ||
1512 | 1694 | ||
1513 | return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); |
1695 | return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); |
Line 1514... | Line -... | ||
1514 | } |
- | |
1515 | - | ||
1516 | static inline int |
- | |
1517 | i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) |
- | |
1518 | { |
- | |
1519 | return obj->madv == I915_MADV_DONTNEED; |
- | |
1520 | } |
1696 | } |
1521 | 1697 | ||
1522 | /* Immediately discard the backing storage */ |
1698 | /* Immediately discard the backing storage */ |
1523 | static void |
1699 | static void |
1524 | i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
1700 | i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
Line 1571... | Line 1747... | ||
1571 | WARN_ON(ret != -EIO); |
1747 | WARN_ON(ret != -EIO); |
1572 | i915_gem_clflush_object(obj, true); |
1748 | i915_gem_clflush_object(obj, true); |
1573 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1749 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1574 | } |
1750 | } |
Line -... | Line 1751... | ||
- | 1751 | ||
1575 | 1752 | i915_gem_gtt_finish_object(obj); |
|
1576 | if (obj->madv == I915_MADV_DONTNEED) |
1753 | if (obj->madv == I915_MADV_DONTNEED) |
Line 1577... | Line 1754... | ||
1577 | obj->dirty = 0; |
1754 | obj->dirty = 0; |
1578 | 1755 | ||
Line 1611... | Line 1788... | ||
1611 | i915_gem_object_invalidate(obj); |
1788 | i915_gem_object_invalidate(obj); |
Line 1612... | Line 1789... | ||
1612 | 1789 | ||
1613 | return 0; |
1790 | return 0; |
Line 1614... | Line -... | ||
1614 | } |
- | |
1615 | - | ||
1616 | - | ||
1617 | - | ||
1618 | - | ||
1619 | - | ||
1620 | - | ||
1621 | 1791 | } |
|
1622 | 1792 | ||
1623 | static int |
1793 | static int |
1624 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) |
1794 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) |
1625 | { |
1795 | { |
- | 1796 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
|
1626 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
1797 | int page_count, i; |
1627 | int page_count, i; |
1798 | struct address_space *mapping; |
1628 | struct sg_table *st; |
1799 | struct sg_table *st; |
1629 | struct scatterlist *sg; |
1800 | struct scatterlist *sg; |
1630 | struct sg_page_iter sg_iter; |
1801 | struct sg_page_iter sg_iter; |
- | 1802 | struct page *page; |
|
1631 | struct page *page; |
1803 | unsigned long last_pfn = 0; /* suppress gcc warning */ |
Line 1632... | Line 1804... | ||
1632 | unsigned long last_pfn = 0; /* suppress gcc warning */ |
1804 | int ret; |
1633 | gfp_t gfp; |
1805 | gfp_t gfp; |
1634 | 1806 | ||
Line 1684... | Line 1856... | ||
1684 | if (!swiotlb_nr_tbl()) |
1856 | if (!swiotlb_nr_tbl()) |
1685 | #endif |
1857 | #endif |
1686 | sg_mark_end(sg); |
1858 | sg_mark_end(sg); |
1687 | obj->pages = st; |
1859 | obj->pages = st; |
Line -... | Line 1860... | ||
- | 1860 | ||
- | 1861 | ret = i915_gem_gtt_prepare_object(obj); |
|
- | 1862 | if (ret) |
|
Line 1688... | Line 1863... | ||
1688 | 1863 | goto err_pages; |
|
1689 | 1864 | ||
1690 | if (obj->tiling_mode != I915_TILING_NONE && |
1865 | if (obj->tiling_mode != I915_TILING_NONE && |
Line 1730... | Line 1905... | ||
1730 | ret = ops->get_pages(obj); |
1905 | ret = ops->get_pages(obj); |
1731 | if (ret) |
1906 | if (ret) |
1732 | return ret; |
1907 | return ret; |
Line 1733... | Line 1908... | ||
1733 | 1908 | ||
- | 1909 | list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); |
|
- | 1910 | ||
- | 1911 | obj->get_page.sg = obj->pages->sgl; |
|
- | 1912 | obj->get_page.last = 0; |
|
1734 | list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); |
1913 | |
1735 | return 0; |
1914 | return 0; |
Line 1736... | Line -... | ||
1736 | } |
- | |
1737 | 1915 | } |
|
1738 | static void |
1916 | |
1739 | i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, |
1917 | void i915_vma_move_to_active(struct i915_vma *vma, |
- | 1918 | struct drm_i915_gem_request *req) |
|
1740 | struct intel_engine_cs *ring) |
1919 | { |
Line 1741... | Line -... | ||
1741 | { |
- | |
1742 | u32 seqno = intel_ring_get_seqno(ring); |
1920 | struct drm_i915_gem_object *obj = vma->obj; |
1743 | - | ||
1744 | BUG_ON(ring == NULL); |
- | |
1745 | if (obj->ring != ring && obj->last_write_seqno) { |
- | |
1746 | /* Keep the seqno relative to the current ring */ |
- | |
Line 1747... | Line 1921... | ||
1747 | obj->last_write_seqno = seqno; |
1921 | struct intel_engine_cs *ring; |
1748 | } |
1922 | |
1749 | obj->ring = ring; |
1923 | ring = i915_gem_request_get_ring(req); |
1750 | 1924 | ||
1751 | /* Add a reference if we're newly entering the active list. */ |
- | |
Line 1752... | Line 1925... | ||
1752 | if (!obj->active) { |
1925 | /* Add a reference if we're newly entering the active list. */ |
- | 1926 | if (obj->active == 0) |
|
Line 1753... | Line 1927... | ||
1753 | drm_gem_object_reference(&obj->base); |
1927 | drm_gem_object_reference(&obj->base); |
1754 | obj->active = 1; |
1928 | obj->active |= intel_ring_flag(ring); |
Line -... | Line 1929... | ||
- | 1929 | ||
1755 | } |
1930 | list_move_tail(&obj->ring_list[ring->id], &ring->active_list); |
1756 | - | ||
1757 | list_move_tail(&obj->ring_list, &ring->active_list); |
1931 | i915_gem_request_assign(&obj->last_read_req[ring->id], req); |
- | 1932 | ||
1758 | 1933 | list_move_tail(&vma->mm_list, &vma->vm->active_list); |
|
- | 1934 | } |
|
1759 | obj->last_read_seqno = seqno; |
1935 | |
- | 1936 | static void |
|
1760 | } |
1937 | i915_gem_object_retire__write(struct drm_i915_gem_object *obj) |
Line 1761... | Line 1938... | ||
1761 | 1938 | { |
|
1762 | void i915_vma_move_to_active(struct i915_vma *vma, |
1939 | RQ_BUG_ON(obj->last_write_req == NULL); |
1763 | struct intel_engine_cs *ring) |
1940 | RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring))); |
1764 | { |
- | |
1765 | list_move_tail(&vma->mm_list, &vma->vm->active_list); |
- | |
1766 | return i915_gem_object_move_to_active(vma->obj, ring); |
1941 | |
Line 1767... | Line 1942... | ||
1767 | } |
1942 | i915_gem_request_assign(&obj->last_write_req, NULL); |
1768 | 1943 | intel_fb_obj_flush(obj, true, ORIGIN_CS); |
|
1769 | static void |
- | |
1770 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
- | |
1771 | { |
- | |
1772 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
- | |
1773 | struct i915_address_space *vm; |
- | |
1774 | struct i915_vma *vma; |
- | |
1775 | - | ||
1776 | BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); |
- | |
Line 1777... | Line 1944... | ||
1777 | BUG_ON(!obj->active); |
1944 | } |
1778 | 1945 | ||
Line 1779... | Line -... | ||
1779 | list_for_each_entry(vm, &dev_priv->vm_list, global_link) { |
- | |
1780 | vma = i915_gem_obj_to_vma(obj, vm); |
1946 | static void |
1781 | if (vma && !list_empty(&vma->mm_list)) |
1947 | i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring) |
Line 1782... | Line 1948... | ||
1782 | list_move_tail(&vma->mm_list, &vm->inactive_list); |
1948 | { |
- | 1949 | struct i915_vma *vma; |
|
- | 1950 | ||
Line -... | Line 1951... | ||
- | 1951 | RQ_BUG_ON(obj->last_read_req[ring] == NULL); |
|
- | 1952 | RQ_BUG_ON(!(obj->active & (1 << ring))); |
|
1783 | } |
1953 | |
- | 1954 | list_del_init(&obj->ring_list[ring]); |
|
1784 | 1955 | i915_gem_request_assign(&obj->last_read_req[ring], NULL); |
|
- | 1956 | ||
Line -... | Line 1957... | ||
- | 1957 | if (obj->last_write_req && obj->last_write_req->ring->id == ring) |
|
1785 | intel_fb_obj_flush(obj, true); |
1958 | i915_gem_object_retire__write(obj); |
- | 1959 | ||
1786 | 1960 | obj->active &= ~(1 << ring); |
|
Line 1787... | Line -... | ||
1787 | list_del_init(&obj->ring_list); |
- | |
1788 | obj->ring = NULL; |
1961 | if (obj->active) |
1789 | - | ||
1790 | obj->last_read_seqno = 0; |
- | |
1791 | obj->last_write_seqno = 0; |
- | |
1792 | obj->base.write_domain = 0; |
- | |
1793 | - | ||
1794 | obj->last_fenced_seqno = 0; |
- | |
1795 | - | ||
1796 | obj->active = 0; |
- | |
1797 | drm_gem_object_unreference(&obj->base); |
1962 | return; |
1798 | 1963 | ||
Line 1799... | Line 1964... | ||
1799 | WARN_ON(i915_verify_lists(dev)); |
1964 | /* Bump our place on the bound list to keep it roughly in LRU order |
1800 | } |
1965 | * so that we don't steal from recently used but inactive objects |
1801 | 1966 | * (unless we are forced to ofc!) |
|
Line 1880... | Line 2045... | ||
1880 | 2045 | ||
1881 | *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; |
2046 | *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; |
1882 | return 0; |
2047 | return 0; |
Line -... | Line 2048... | ||
- | 2048 | } |
|
- | 2049 | ||
- | 2050 | /* |
|
1883 | } |
2051 | * NB: This function is not allowed to fail. Doing so would mean the the |
- | 2052 | * request is not being tracked for completion but the work itself is |
|
1884 | 2053 | * going to happen on the hardware. This would be a Bad Thing(tm). |
|
1885 | int __i915_add_request(struct intel_engine_cs *ring, |
2054 | */ |
1886 | struct drm_file *file, |
2055 | void __i915_add_request(struct drm_i915_gem_request *request, |
1887 | struct drm_i915_gem_object *obj, |
2056 | struct drm_i915_gem_object *obj, |
1888 | u32 *out_seqno) |
2057 | bool flush_caches) |
1889 | { |
2058 | { |
1890 | struct drm_i915_private *dev_priv = ring->dev->dev_private; |
2059 | struct intel_engine_cs *ring; |
1891 | struct drm_i915_gem_request *request; |
2060 | struct drm_i915_private *dev_priv; |
1892 | struct intel_ringbuffer *ringbuf; |
2061 | struct intel_ringbuffer *ringbuf; |
Line 1893... | Line -... | ||
1893 | u32 request_ring_position, request_start; |
- | |
1894 | int ret; |
2062 | u32 request_start; |
1895 | 2063 | int ret; |
|
Line 1896... | Line 2064... | ||
1896 | request = ring->preallocated_lazy_request; |
2064 | |
1897 | if (WARN_ON(request == NULL)) |
2065 | if (WARN_ON(request == NULL)) |
1898 | return -ENOMEM; |
2066 | return; |
- | 2067 | ||
- | 2068 | ring = request->ring; |
|
- | 2069 | dev_priv = ring->dev->dev_private; |
|
- | 2070 | ringbuf = request->ringbuf; |
|
- | 2071 | ||
1899 | 2072 | /* |
|
1900 | if (i915.enable_execlists) { |
2073 | * To ensure that this call will not fail, space for its emissions |
Line 1901... | Line 2074... | ||
1901 | struct intel_context *ctx = request->ctx; |
2074 | * should already have been reserved in the ring buffer. Let the ring |
1902 | ringbuf = ctx->engine[ring->id].ringbuf; |
2075 | * know that it is time to use that space up. |
1903 | } else |
2076 | */ |
1904 | ringbuf = ring->buffer; |
2077 | intel_ring_reserved_space_use(ringbuf); |
1905 | 2078 | ||
1906 | request_start = intel_ring_get_tail(ringbuf); |
2079 | request_start = intel_ring_get_tail(ringbuf); |
1907 | /* |
2080 | /* |
1908 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
2081 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
- | 2082 | * after having emitted the batchbuffer command. Hence we need to fix |
|
1909 | * after having emitted the batchbuffer command. Hence we need to fix |
2083 | * things up similar to emitting the lazy request. The difference here |
1910 | * things up similar to emitting the lazy request. The difference here |
2084 | * is that the flush _must_ happen before the next request, no matter |
1911 | * is that the flush _must_ happen before the next request, no matter |
- | |
1912 | * what. |
- | |
1913 | */ |
2085 | * what. |
1914 | if (i915.enable_execlists) { |
2086 | */ |
1915 | ret = logical_ring_flush_all_caches(ringbuf); |
2087 | if (flush_caches) { |
1916 | if (ret) |
2088 | if (i915.enable_execlists) |
1917 | return ret; |
2089 | ret = logical_ring_flush_all_caches(request); |
Line 1918... | Line 2090... | ||
1918 | } else { |
2090 | else |
1919 | ret = intel_ring_flush_all_caches(ring); |
2091 | ret = intel_ring_flush_all_caches(request); |
1920 | if (ret) |
2092 | /* Not allowed to fail! */ |
1921 | return ret; |
2093 | WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret); |
1922 | } |
2094 | } |
1923 | 2095 | ||
Line 1924... | Line 2096... | ||
1924 | /* Record the position of the start of the request so that |
2096 | /* Record the position of the start of the request so that |
1925 | * should we detect the updated seqno part-way through the |
2097 | * should we detect the updated seqno part-way through the |
1926 | * GPU processing the request, we never over-estimate the |
- | |
1927 | * position of the head. |
- | |
1928 | */ |
2098 | * GPU processing the request, we never over-estimate the |
1929 | request_ring_position = intel_ring_get_tail(ringbuf); |
2099 | * position of the head. |
1930 | 2100 | */ |
|
1931 | if (i915.enable_execlists) { |
2101 | request->postfix = intel_ring_get_tail(ringbuf); |
1932 | ret = ring->emit_request(ringbuf); |
2102 | |
- | 2103 | if (i915.enable_execlists) |
|
- | 2104 | ret = ring->emit_request(request); |
|
Line 1933... | Line -... | ||
1933 | if (ret) |
- | |
1934 | return ret; |
- | |
1935 | } else { |
2105 | else { |
1936 | ret = ring->add_request(ring); |
- | |
Line 1937... | Line 2106... | ||
1937 | if (ret) |
2106 | ret = ring->add_request(request); |
1938 | return ret; |
2107 | |
1939 | } |
2108 | request->tail = intel_ring_get_tail(ringbuf); |
1940 | 2109 | } |
|
1941 | request->seqno = intel_ring_get_seqno(ring); |
2110 | /* Not allowed to fail! */ |
1942 | request->ring = ring; |
2111 | WARN(ret, "emit|add_request failed: %d!\n", ret); |
1943 | request->head = request_start; |
2112 | |
Line 1944... | Line -... | ||
1944 | request->tail = request_ring_position; |
- | |
1945 | - | ||
1946 | /* Whilst this request exists, batch_obj will be on the |
- | |
1947 | * active_list, and so will hold the active reference. Only when this |
- | |
1948 | * request is retired will the the batch_obj be moved onto the |
- | |
1949 | * inactive_list and lose its active reference. Hence we do not need |
- | |
1950 | * to explicitly hold another reference here. |
- | |
1951 | */ |
- | |
1952 | request->batch_obj = obj; |
- | |
1953 | 2113 | request->head = request_start; |
|
- | 2114 | ||
- | 2115 | /* Whilst this request exists, batch_obj will be on the |
|
1954 | if (!i915.enable_execlists) { |
2116 | * active_list, and so will hold the active reference. Only when this |
1955 | /* Hold a reference to the current context so that we can inspect |
- | |
1956 | * it later in case a hangcheck error event fires. |
- | |
1957 | */ |
- | |
1958 | request->ctx = ring->last_context; |
- | |
1959 | if (request->ctx) |
- | |
1960 | i915_gem_context_reference(request->ctx); |
- | |
1961 | } |
- | |
1962 | - | ||
1963 | request->emitted_jiffies = jiffies; |
- | |
1964 | list_add_tail(&request->list, &ring->request_list); |
- | |
1965 | request->file_priv = NULL; |
- | |
Line 1966... | Line 2117... | ||
1966 | 2117 | * request is retired will the the batch_obj be moved onto the |
|
1967 | if (file) { |
- | |
1968 | struct drm_i915_file_private *file_priv = file->driver_priv; |
- | |
Line 1969... | Line 2118... | ||
1969 | 2118 | * inactive_list and lose its active reference. Hence we do not need |
|
Line 1970... | Line 2119... | ||
1970 | spin_lock(&file_priv->mm.lock); |
2119 | * to explicitly hold another reference here. |
1971 | request->file_priv = file_priv; |
2120 | */ |
1972 | list_add_tail(&request->client_list, |
2121 | request->batch_obj = obj; |
1973 | &file_priv->mm.request_list); |
2122 | |
Line 1974... | Line -... | ||
1974 | spin_unlock(&file_priv->mm.lock); |
- | |
1975 | } |
- | |
1976 | - | ||
1977 | trace_i915_gem_request_add(ring, request->seqno); |
- | |
1978 | ring->outstanding_lazy_seqno = 0; |
- | |
1979 | ring->preallocated_lazy_request = NULL; |
- | |
1980 | - | ||
1981 | // i915_queue_hangcheck(ring->dev); |
- | |
1982 | 2123 | request->emitted_jiffies = jiffies; |
|
1983 | queue_delayed_work(dev_priv->wq, |
- | |
1984 | &dev_priv->mm.retire_work, |
- | |
1985 | round_jiffies_up_relative(HZ)); |
- | |
1986 | intel_mark_busy(dev_priv->dev); |
- | |
1987 | - | ||
1988 | if (out_seqno) |
2124 | request->previous_seqno = ring->last_submitted_seqno; |
1989 | *out_seqno = request->seqno; |
- | |
1990 | return 0; |
- | |
1991 | } |
2125 | ring->last_submitted_seqno = request->seqno; |
Line 1992... | Line 2126... | ||
1992 | 2126 | list_add_tail(&request->list, &ring->request_list); |
|
1993 | static inline void |
2127 | |
1994 | i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) |
2128 | trace_i915_gem_request_add(request); |
Line 2012... | Line 2146... | ||
2012 | elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts; |
2146 | elapsed = GetTimerTicks()/100 - ctx->hang_stats.guilty_ts; |
Line 2013... | Line 2147... | ||
2013 | 2147 | ||
2014 | if (ctx->hang_stats.banned) |
2148 | if (ctx->hang_stats.banned) |
Line -... | Line 2149... | ||
- | 2149 | return true; |
|
2015 | return true; |
2150 | |
2016 | 2151 | if (ctx->hang_stats.ban_period_seconds && |
|
2017 | if (elapsed <= DRM_I915_CTX_BAN_PERIOD) { |
2152 | elapsed <= ctx->hang_stats.ban_period_seconds) { |
2018 | if (!i915_gem_context_is_default(ctx)) { |
2153 | if (!i915_gem_context_is_default(ctx)) { |
2019 | DRM_DEBUG("context hanging too fast, banning!\n"); |
2154 | DRM_DEBUG("context hanging too fast, banning!\n"); |
2020 | return true; |
2155 | return true; |
Line 2046... | Line 2181... | ||
2046 | } else { |
2181 | } else { |
2047 | hs->batch_pending++; |
2182 | hs->batch_pending++; |
2048 | } |
2183 | } |
2049 | } |
2184 | } |
Line 2050... | Line 2185... | ||
2050 | 2185 | ||
2051 | static void i915_gem_free_request(struct drm_i915_gem_request *request) |
2186 | void i915_gem_request_free(struct kref *req_ref) |
- | 2187 | { |
|
- | 2188 | struct drm_i915_gem_request *req = container_of(req_ref, |
|
2052 | { |
2189 | typeof(*req), ref); |
Line 2053... | Line 2190... | ||
2053 | struct intel_context *ctx = request->ctx; |
2190 | struct intel_context *ctx = req->ctx; |
2054 | 2191 | ||
Line 2055... | Line 2192... | ||
2055 | list_del(&request->list); |
2192 | if (req->file_priv) |
2056 | i915_gem_request_remove_from_client(request); |
2193 | i915_gem_request_remove_from_client(req); |
2057 | - | ||
2058 | if (ctx) { |
- | |
2059 | if (i915.enable_execlists) { |
2194 | |
2060 | struct intel_engine_cs *ring = request->ring; |
2195 | if (ctx) { |
2061 | 2196 | if (i915.enable_execlists) { |
|
- | 2197 | if (ctx != req->ring->default_context) |
|
2062 | if (ctx != ring->default_context) |
2198 | intel_lr_context_unpin(req); |
2063 | intel_lr_context_unpin(ring, ctx); |
2199 | } |
- | 2200 | ||
- | 2201 | i915_gem_context_unreference(ctx); |
|
- | 2202 | } |
|
- | 2203 | ||
- | 2204 | kfree(req); |
|
- | 2205 | } |
|
- | 2206 | ||
- | 2207 | int i915_gem_request_alloc(struct intel_engine_cs *ring, |
|
- | 2208 | struct intel_context *ctx, |
|
- | 2209 | struct drm_i915_gem_request **req_out) |
|
- | 2210 | { |
|
- | 2211 | struct drm_i915_private *dev_priv = to_i915(ring->dev); |
|
- | 2212 | struct drm_i915_gem_request *req; |
|
- | 2213 | int ret; |
|
- | 2214 | ||
- | 2215 | if (!req_out) |
|
- | 2216 | return -EINVAL; |
|
- | 2217 | ||
- | 2218 | *req_out = NULL; |
|
- | 2219 | ||
- | 2220 | // req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL); |
|
- | 2221 | req = kzalloc(sizeof(*req),0); |
|
- | 2222 | if (req == NULL) |
|
- | 2223 | return -ENOMEM; |
|
- | 2224 | ||
- | 2225 | ret = i915_gem_get_seqno(ring->dev, &req->seqno); |
|
- | 2226 | if (ret) |
|
- | 2227 | goto err; |
|
- | 2228 | ||
- | 2229 | kref_init(&req->ref); |
|
- | 2230 | req->i915 = dev_priv; |
|
- | 2231 | req->ring = ring; |
|
- | 2232 | req->ctx = ctx; |
|
- | 2233 | i915_gem_context_reference(req->ctx); |
|
- | 2234 | ||
- | 2235 | if (i915.enable_execlists) |
|
- | 2236 | ret = intel_logical_ring_alloc_request_extras(req); |
|
- | 2237 | else |
|
- | 2238 | ret = intel_ring_alloc_request_extras(req); |
|
- | 2239 | if (ret) { |
|
- | 2240 | i915_gem_context_unreference(req->ctx); |
|
- | 2241 | goto err; |
|
- | 2242 | } |
|
- | 2243 | ||
- | 2244 | /* |
|
- | 2245 | * Reserve space in the ring buffer for all the commands required to |
|
- | 2246 | * eventually emit this request. This is to guarantee that the |
|
- | 2247 | * i915_add_request() call can't fail. Note that the reserve may need |
|
- | 2248 | * to be redone if the request is not actually submitted straight |
|
- | 2249 | * away, e.g. because a GPU scheduler has deferred it. |
|
- | 2250 | */ |
|
- | 2251 | if (i915.enable_execlists) |
|
- | 2252 | ret = intel_logical_ring_reserve_space(req); |
|
- | 2253 | else |
|
- | 2254 | ret = intel_ring_reserve_space(req); |
|
- | 2255 | if (ret) { |
|
- | 2256 | /* |
|
- | 2257 | * At this point, the request is fully allocated even if not |
|
- | 2258 | * fully prepared. Thus it can be cleaned up using the proper |
|
- | 2259 | * free code. |
|
- | 2260 | */ |
|
- | 2261 | i915_gem_request_cancel(req); |
|
- | 2262 | return ret; |
|
- | 2263 | } |
|
- | 2264 | ||
- | 2265 | *req_out = req; |
|
2064 | } |
2266 | return 0; |
- | 2267 | ||
- | 2268 | err: |
|
- | 2269 | kfree(req); |
|
- | 2270 | return ret; |
|
- | 2271 | } |
|
- | 2272 | ||
- | 2273 | void i915_gem_request_cancel(struct drm_i915_gem_request *req) |
|
- | 2274 | { |
|
2065 | i915_gem_context_unreference(ctx); |
2275 | intel_ring_reserved_space_cancel(req->ringbuf); |
Line 2066... | Line 2276... | ||
2066 | } |
2276 | |
2067 | kfree(request); |
2277 | i915_gem_request_unreference(req); |
2068 | } |
2278 | } |
2069 | 2279 | ||
2070 | struct drm_i915_gem_request * |
- | |
2071 | i915_gem_find_active_request(struct intel_engine_cs *ring) |
- | |
2072 | { |
- | |
Line 2073... | Line 2280... | ||
2073 | struct drm_i915_gem_request *request; |
2280 | struct drm_i915_gem_request * |
2074 | u32 completed_seqno; |
2281 | i915_gem_find_active_request(struct intel_engine_cs *ring) |
2075 | 2282 | { |
|
Line 2076... | Line 2283... | ||
2076 | completed_seqno = ring->get_seqno(ring, false); |
2283 | struct drm_i915_gem_request *request; |
2077 | 2284 | ||
Line 2110... | Line 2317... | ||
2110 | while (!list_empty(&ring->active_list)) { |
2317 | while (!list_empty(&ring->active_list)) { |
2111 | struct drm_i915_gem_object *obj; |
2318 | struct drm_i915_gem_object *obj; |
Line 2112... | Line 2319... | ||
2112 | 2319 | ||
2113 | obj = list_first_entry(&ring->active_list, |
2320 | obj = list_first_entry(&ring->active_list, |
2114 | struct drm_i915_gem_object, |
2321 | struct drm_i915_gem_object, |
Line 2115... | Line 2322... | ||
2115 | ring_list); |
2322 | ring_list[ring->id]); |
2116 | 2323 | ||
Line 2117... | Line 2324... | ||
2117 | i915_gem_object_move_to_inactive(obj); |
2324 | i915_gem_object_retire__read(obj, ring->id); |
2118 | } |
2325 | } |
2119 | 2326 | ||
2120 | /* |
2327 | /* |
2121 | * Clear the execlists queue up before freeing the requests, as those |
2328 | * Clear the execlists queue up before freeing the requests, as those |
2122 | * are the ones that keep the context and ringbuffer backing objects |
2329 | * are the ones that keep the context and ringbuffer backing objects |
2123 | * pinned in place. |
2330 | * pinned in place. |
Line 2124... | Line 2331... | ||
2124 | */ |
2331 | */ |
2125 | while (!list_empty(&ring->execlist_queue)) { |
2332 | while (!list_empty(&ring->execlist_queue)) { |
2126 | struct intel_ctx_submit_request *submit_req; |
2333 | struct drm_i915_gem_request *submit_req; |
2127 | 2334 | ||
- | 2335 | submit_req = list_first_entry(&ring->execlist_queue, |
|
2128 | submit_req = list_first_entry(&ring->execlist_queue, |
2336 | struct drm_i915_gem_request, |
2129 | struct intel_ctx_submit_request, |
2337 | execlist_link); |
- | 2338 | list_del(&submit_req->execlist_link); |
|
2130 | execlist_link); |
2339 | |
2131 | list_del(&submit_req->execlist_link); |
2340 | if (submit_req->ctx != ring->default_context) |
Line 2132... | Line 2341... | ||
2132 | intel_runtime_pm_put(dev_priv); |
2341 | intel_lr_context_unpin(submit_req); |
2133 | i915_gem_context_unreference(submit_req->ctx); |
2342 | |
2134 | kfree(submit_req); |
2343 | i915_gem_request_unreference(submit_req); |
Line 2146... | Line 2355... | ||
2146 | 2355 | ||
2147 | request = list_first_entry(&ring->request_list, |
2356 | request = list_first_entry(&ring->request_list, |
2148 | struct drm_i915_gem_request, |
2357 | struct drm_i915_gem_request, |
Line 2149... | Line 2358... | ||
2149 | list); |
2358 | list); |
2150 | - | ||
2151 | i915_gem_free_request(request); |
- | |
2152 | } |
- | |
2153 | - | ||
2154 | /* These may not have been flush before the reset, do so now */ |
- | |
2155 | kfree(ring->preallocated_lazy_request); |
- | |
2156 | ring->preallocated_lazy_request = NULL; |
- | |
2157 | ring->outstanding_lazy_seqno = 0; |
- | |
2158 | } |
- | |
2159 | - | ||
2160 | void i915_gem_restore_fences(struct drm_device *dev) |
- | |
2161 | { |
- | |
2162 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2163 | int i; |
- | |
2164 | - | ||
2165 | for (i = 0; i < dev_priv->num_fence_regs; i++) { |
- | |
2166 | struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
- | |
2167 | - | ||
2168 | /* |
- | |
2169 | * Commit delayed tiling changes if we have an object still |
- | |
2170 | * attached to the fence, otherwise just clear the fence. |
- | |
2171 | */ |
- | |
2172 | if (reg->obj) { |
- | |
2173 | i915_gem_object_update_fence(reg->obj, reg, |
- | |
2174 | reg->obj->tiling_mode); |
- | |
2175 | } else { |
- | |
2176 | i915_gem_write_fence(dev, i, NULL); |
2359 | |
2177 | } |
2360 | i915_gem_request_retire(request); |
Line 2178... | Line 2361... | ||
2178 | } |
2361 | } |
2179 | } |
2362 | } |
Line 2196... | Line 2379... | ||
2196 | i915_gem_reset_ring_cleanup(dev_priv, ring); |
2379 | i915_gem_reset_ring_cleanup(dev_priv, ring); |
Line 2197... | Line 2380... | ||
2197 | 2380 | ||
Line 2198... | Line 2381... | ||
2198 | i915_gem_context_reset(dev); |
2381 | i915_gem_context_reset(dev); |
- | 2382 | ||
- | 2383 | i915_gem_restore_fences(dev); |
|
2199 | 2384 | ||
Line 2200... | Line 2385... | ||
2200 | i915_gem_restore_fences(dev); |
2385 | WARN_ON(i915_verify_lists(dev)); |
2201 | } |
2386 | } |
2202 | 2387 | ||
2203 | /** |
2388 | /** |
2204 | * This function clears the request list as sequence numbers are passed. |
2389 | * This function clears the request list as sequence numbers are passed. |
2205 | */ |
2390 | */ |
2206 | void |
- | |
2207 | i915_gem_retire_requests_ring(struct intel_engine_cs *ring) |
- | |
2208 | { |
- | |
2209 | uint32_t seqno; |
- | |
2210 | - | ||
2211 | if (list_empty(&ring->request_list)) |
2391 | void |
Line 2212... | Line 2392... | ||
2212 | return; |
2392 | i915_gem_retire_requests_ring(struct intel_engine_cs *ring) |
2213 | - | ||
2214 | WARN_ON(i915_verify_lists(ring->dev)); |
2393 | { |
2215 | 2394 | WARN_ON(i915_verify_lists(ring->dev)); |
|
2216 | seqno = ring->get_seqno(ring, true); |
2395 | |
2217 | 2396 | /* Retire requests first as we use it above for the early return. |
|
2218 | /* Move any buffers on the active list that are no longer referenced |
- | |
2219 | * by the ringbuffer to the flushing/inactive lists as appropriate, |
- | |
2220 | * before we free the context associated with the requests. |
- | |
2221 | */ |
- | |
2222 | while (!list_empty(&ring->active_list)) { |
- | |
2223 | struct drm_i915_gem_object *obj; |
- | |
2224 | - | ||
2225 | obj = list_first_entry(&ring->active_list, |
- | |
2226 | struct drm_i915_gem_object, |
- | |
2227 | ring_list); |
- | |
2228 | - | ||
2229 | if (!i915_seqno_passed(seqno, obj->last_read_seqno)) |
- | |
2230 | break; |
- | |
2231 | - | ||
2232 | i915_gem_object_move_to_inactive(obj); |
2397 | * If we retire requests last, we may use a later seqno and so clear |
2233 | } |
2398 | * the requests lists without clearing the active list, leading to |
2234 | - | ||
Line 2235... | Line 2399... | ||
2235 | 2399 | * confusion. |
|
2236 | while (!list_empty(&ring->request_list)) { |
2400 | */ |
2237 | struct drm_i915_gem_request *request; |
2401 | while (!list_empty(&ring->request_list)) { |
Line 2238... | Line 2402... | ||
2238 | struct intel_ringbuffer *ringbuf; |
2402 | struct drm_i915_gem_request *request; |
2239 | 2403 | ||
Line 2240... | Line 2404... | ||
2240 | request = list_first_entry(&ring->request_list, |
2404 | request = list_first_entry(&ring->request_list, |
- | 2405 | struct drm_i915_gem_request, |
|
Line 2241... | Line 2406... | ||
2241 | struct drm_i915_gem_request, |
2406 | list); |
2242 | list); |
2407 | |
2243 | 2408 | if (!i915_gem_request_completed(request, true)) |
|
2244 | if (!i915_seqno_passed(seqno, request->seqno)) |
- | |
2245 | break; |
2409 | break; |
2246 | 2410 | ||
2247 | trace_i915_gem_request_retire(ring, request->seqno); |
2411 | i915_gem_request_retire(request); |
2248 | - | ||
2249 | /* This is one of the few common intersection points |
- | |
2250 | * between legacy ringbuffer submission and execlists: |
- | |
Line 2251... | Line 2412... | ||
2251 | * we need to tell them apart in order to find the correct |
2412 | } |
2252 | * ringbuffer to which the request belongs to. |
2413 | |
2253 | */ |
- | |
2254 | if (i915.enable_execlists) { |
2414 | /* Move any buffers on the active list that are no longer referenced |
2255 | struct intel_context *ctx = request->ctx; |
- | |
2256 | ringbuf = ctx->engine[ring->id].ringbuf; |
- | |
Line -... | Line 2415... | ||
- | 2415 | * by the ringbuffer to the flushing/inactive lists as appropriate, |
|
- | 2416 | * before we free the context associated with the requests. |
|
- | 2417 | */ |
|
2257 | } else |
2418 | while (!list_empty(&ring->active_list)) { |
2258 | ringbuf = ring->buffer; |
2419 | struct drm_i915_gem_object *obj; |
Line 2259... | Line 2420... | ||
2259 | 2420 | ||
2260 | /* We know the GPU must have read the request to have |
2421 | obj = list_first_entry(&ring->active_list, |
2261 | * sent us the seqno + interrupt, so use the position |
2422 | struct drm_i915_gem_object, |
2262 | * of tail of the request to update the last known position |
2423 | ring_list[ring->id]); |
2263 | * of the GPU head. |
2424 | |
Line 2264... | Line 2425... | ||
2264 | */ |
2425 | if (!list_empty(&obj->last_read_req[ring->id]->list)) |
2265 | ringbuf->last_retired_head = request->tail; |
2426 | break; |
Line 2328... | Line 2489... | ||
2328 | static void |
2489 | static void |
2329 | i915_gem_idle_work_handler(struct work_struct *work) |
2490 | i915_gem_idle_work_handler(struct work_struct *work) |
2330 | { |
2491 | { |
2331 | struct drm_i915_private *dev_priv = |
2492 | struct drm_i915_private *dev_priv = |
2332 | container_of(work, typeof(*dev_priv), mm.idle_work.work); |
2493 | container_of(work, typeof(*dev_priv), mm.idle_work.work); |
- | 2494 | struct drm_device *dev = dev_priv->dev; |
|
- | 2495 | struct intel_engine_cs *ring; |
|
- | 2496 | int i; |
|
Line -... | Line 2497... | ||
- | 2497 | ||
- | 2498 | for_each_ring(ring, dev_priv, i) |
|
- | 2499 | if (!list_empty(&ring->request_list)) |
|
- | 2500 | return; |
|
2333 | 2501 | ||
- | 2502 | intel_mark_idle(dev); |
|
- | 2503 | ||
- | 2504 | if (mutex_trylock(&dev->struct_mutex)) { |
|
- | 2505 | struct intel_engine_cs *ring; |
|
- | 2506 | int i; |
|
- | 2507 | ||
- | 2508 | for_each_ring(ring, dev_priv, i) |
|
- | 2509 | i915_gem_batch_pool_fini(&ring->batch_pool); |
|
- | 2510 | ||
- | 2511 | mutex_unlock(&dev->struct_mutex); |
|
2334 | intel_mark_idle(dev_priv->dev); |
2512 | } |
Line 2335... | Line 2513... | ||
2335 | } |
2513 | } |
2336 | 2514 | ||
2337 | /** |
2515 | /** |
2338 | * Ensures that an object will eventually get non-busy by flushing any required |
2516 | * Ensures that an object will eventually get non-busy by flushing any required |
2339 | * write domains, emitting any outstanding lazy request and retiring and |
2517 | * write domains, emitting any outstanding lazy request and retiring and |
2340 | * completed requests. |
2518 | * completed requests. |
2341 | */ |
2519 | */ |
2342 | static int |
2520 | static int |
2343 | i915_gem_object_flush_active(struct drm_i915_gem_object *obj) |
2521 | i915_gem_object_flush_active(struct drm_i915_gem_object *obj) |
Line 2344... | Line 2522... | ||
2344 | { |
2522 | { |
- | 2523 | int i; |
|
- | 2524 | ||
- | 2525 | if (!obj->active) |
|
- | 2526 | return 0; |
|
- | 2527 | ||
2345 | int ret; |
2528 | for (i = 0; i < I915_NUM_RINGS; i++) { |
2346 | 2529 | struct drm_i915_gem_request *req; |
|
2347 | if (obj->active) { |
2530 | |
Line -... | Line 2531... | ||
- | 2531 | req = obj->last_read_req[i]; |
|
- | 2532 | if (req == NULL) |
|
- | 2533 | continue; |
|
- | 2534 | ||
- | 2535 | if (list_empty(&req->list)) |
|
- | 2536 | goto retire; |
|
2348 | ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); |
2537 | |
- | 2538 | if (i915_gem_request_completed(req, true)) { |
|
2349 | if (ret) |
2539 | __i915_gem_request_retire__upto(req); |
Line 2350... | Line 2540... | ||
2350 | return ret; |
2540 | retire: |
2351 | 2541 | i915_gem_object_retire__read(obj, i); |
|
Line 2381... | Line 2571... | ||
2381 | i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
2571 | i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
2382 | { |
2572 | { |
2383 | struct drm_i915_private *dev_priv = dev->dev_private; |
2573 | struct drm_i915_private *dev_priv = dev->dev_private; |
2384 | struct drm_i915_gem_wait *args = data; |
2574 | struct drm_i915_gem_wait *args = data; |
2385 | struct drm_i915_gem_object *obj; |
2575 | struct drm_i915_gem_object *obj; |
2386 | struct intel_engine_cs *ring = NULL; |
2576 | struct drm_i915_gem_request *req[I915_NUM_RINGS]; |
2387 | unsigned reset_counter; |
2577 | unsigned reset_counter; |
2388 | u32 seqno = 0; |
2578 | int i, n = 0; |
2389 | int ret = 0; |
2579 | int ret; |
Line 2390... | Line 2580... | ||
2390 | 2580 | ||
2391 | if (args->flags != 0) |
2581 | if (args->flags != 0) |
Line 2392... | Line 2582... | ||
2392 | return -EINVAL; |
2582 | return -EINVAL; |
Line 2404... | Line 2594... | ||
2404 | /* Need to make sure the object gets inactive eventually. */ |
2594 | /* Need to make sure the object gets inactive eventually. */ |
2405 | ret = i915_gem_object_flush_active(obj); |
2595 | ret = i915_gem_object_flush_active(obj); |
2406 | if (ret) |
2596 | if (ret) |
2407 | goto out; |
2597 | goto out; |
Line 2408... | Line 2598... | ||
2408 | 2598 | ||
2409 | if (obj->active) { |
- | |
2410 | seqno = obj->last_read_seqno; |
- | |
2411 | ring = obj->ring; |
- | |
2412 | } |
- | |
2413 | - | ||
2414 | if (seqno == 0) |
2599 | if (!obj->active) |
Line 2415... | Line 2600... | ||
2415 | goto out; |
2600 | goto out; |
2416 | 2601 | ||
2417 | /* Do this after OLR check to make sure we make forward progress polling |
2602 | /* Do this after OLR check to make sure we make forward progress polling |
2418 | * on this IOCTL with a timeout <=0 (like busy ioctl) |
2603 | * on this IOCTL with a timeout == 0 (like busy ioctl) |
2419 | */ |
2604 | */ |
2420 | if (args->timeout_ns <= 0) { |
2605 | if (args->timeout_ns == 0) { |
2421 | ret = -ETIME; |
2606 | ret = -ETIME; |
Line 2422... | Line 2607... | ||
2422 | goto out; |
2607 | goto out; |
2423 | } |
2608 | } |
- | 2609 | ||
- | 2610 | drm_gem_object_unreference(&obj->base); |
|
- | 2611 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
|
- | 2612 | ||
- | 2613 | for (i = 0; i < I915_NUM_RINGS; i++) { |
|
- | 2614 | if (obj->last_read_req[i] == NULL) |
|
- | 2615 | continue; |
|
- | 2616 | ||
2424 | 2617 | req[n++] = i915_gem_request_reference(obj->last_read_req[i]); |
|
Line -... | Line 2618... | ||
- | 2618 | } |
|
- | 2619 | ||
2425 | drm_gem_object_unreference(&obj->base); |
2620 | mutex_unlock(&dev->struct_mutex); |
- | 2621 | ||
2426 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
2622 | for (i = 0; i < n; i++) { |
- | 2623 | if (ret == 0) |
|
- | 2624 | ret = __i915_wait_request(req[i], reset_counter, true, |
|
- | 2625 | args->timeout_ns > 0 ? &args->timeout_ns : NULL, |
|
Line 2427... | Line 2626... | ||
2427 | mutex_unlock(&dev->struct_mutex); |
2626 | file->driver_priv); |
2428 | 2627 | i915_gem_request_unreference__unlocked(req[i]); |
|
2429 | return __i915_wait_seqno(ring, seqno, reset_counter, true, |
2628 | } |
2430 | &args->timeout_ns, file->driver_priv); |
2629 | return ret; |
2431 | 2630 | ||
Line -... | Line 2631... | ||
- | 2631 | out: |
|
- | 2632 | drm_gem_object_unreference(&obj->base); |
|
- | 2633 | mutex_unlock(&dev->struct_mutex); |
|
- | 2634 | return ret; |
|
- | 2635 | } |
|
- | 2636 | ||
- | 2637 | static int |
|
- | 2638 | __i915_gem_object_sync(struct drm_i915_gem_object *obj, |
|
- | 2639 | struct intel_engine_cs *to, |
|
- | 2640 | struct drm_i915_gem_request *from_req, |
|
- | 2641 | struct drm_i915_gem_request **to_req) |
|
- | 2642 | { |
|
- | 2643 | struct intel_engine_cs *from; |
|
- | 2644 | int ret; |
|
- | 2645 | ||
- | 2646 | from = i915_gem_request_get_ring(from_req); |
|
- | 2647 | if (to == from) |
|
- | 2648 | return 0; |
|
- | 2649 | ||
- | 2650 | if (i915_gem_request_completed(from_req, true)) |
|
- | 2651 | return 0; |
|
- | 2652 | ||
- | 2653 | if (!i915_semaphore_is_enabled(obj->base.dev)) { |
|
- | 2654 | struct drm_i915_private *i915 = to_i915(obj->base.dev); |
|
- | 2655 | ret = __i915_wait_request(from_req, |
|
- | 2656 | atomic_read(&i915->gpu_error.reset_counter), |
|
- | 2657 | i915->mm.interruptible, |
|
- | 2658 | NULL, |
|
- | 2659 | &i915->rps.semaphores); |
|
- | 2660 | if (ret) |
|
- | 2661 | return ret; |
|
- | 2662 | ||
- | 2663 | i915_gem_object_retire_request(obj, from_req); |
|
- | 2664 | } else { |
|
- | 2665 | int idx = intel_ring_sync_index(from, to); |
|
- | 2666 | u32 seqno = i915_gem_request_get_seqno(from_req); |
|
- | 2667 | ||
- | 2668 | WARN_ON(!to_req); |
|
- | 2669 | ||
- | 2670 | if (seqno <= from->semaphore.sync_seqno[idx]) |
|
- | 2671 | return 0; |
|
- | 2672 | ||
- | 2673 | if (*to_req == NULL) { |
|
- | 2674 | ret = i915_gem_request_alloc(to, to->default_context, to_req); |
|
- | 2675 | if (ret) |
|
- | 2676 | return ret; |
|
- | 2677 | } |
|
- | 2678 | ||
- | 2679 | trace_i915_gem_ring_sync_to(*to_req, from, from_req); |
|
- | 2680 | ret = to->semaphore.sync_to(*to_req, from, seqno); |
|
- | 2681 | if (ret) |
|
- | 2682 | return ret; |
|
- | 2683 | ||
- | 2684 | /* We use last_read_req because sync_to() |
|
- | 2685 | * might have just caused seqno wrap under |
|
- | 2686 | * the radar. |
|
- | 2687 | */ |
|
- | 2688 | from->semaphore.sync_seqno[idx] = |
|
2432 | out: |
2689 | i915_gem_request_get_seqno(obj->last_read_req[from->id]); |
2433 | drm_gem_object_unreference(&obj->base); |
2690 | } |
2434 | mutex_unlock(&dev->struct_mutex); |
2691 | |
2435 | return ret; |
2692 | return 0; |
2436 | } |
2693 | } |
- | 2694 | ||
- | 2695 | /** |
|
- | 2696 | * i915_gem_object_sync - sync an object to a ring. |
|
2437 | 2697 | * |
|
2438 | /** |
2698 | * @obj: object which may be in use on another ring. |
2439 | * i915_gem_object_sync - sync an object to a ring. |
2699 | * @to: ring we wish to use the object on. May be NULL. |
2440 | * |
2700 | * @to_req: request we wish to use the object for. See below. |
- | 2701 | * This will be allocated and returned if a request is |
|
- | 2702 | * required but not passed in. |
|
- | 2703 | * |
|
- | 2704 | * This code is meant to abstract object synchronization with the GPU. |
|
- | 2705 | * Calling with NULL implies synchronizing the object with the CPU |
|
- | 2706 | * rather than a particular GPU ring. Conceptually we serialise writes |
|
- | 2707 | * between engines inside the GPU. We only allow one engine to write |
|
- | 2708 | * into a buffer at any time, but multiple readers. To ensure each has |
|
- | 2709 | * a coherent view of memory, we must: |
|
- | 2710 | * |
|
- | 2711 | * - If there is an outstanding write request to the object, the new |
|
- | 2712 | * request must wait for it to complete (either CPU or in hw, requests |
|
- | 2713 | * on the same ring will be naturally ordered). |
|
- | 2714 | * |
|
- | 2715 | * - If we are a write request (pending_write_domain is set), the new |
|
- | 2716 | * request must wait for outstanding read requests to complete. |
|
- | 2717 | * |
|
- | 2718 | * For CPU synchronisation (NULL to) no request is required. For syncing with |
|
- | 2719 | * rings to_req must be non-NULL. However, a request does not have to be |
|
- | 2720 | * pre-allocated. If *to_req is NULL and sync commands will be emitted then a |
|
2441 | * @obj: object which may be in use on another ring. |
2721 | * request will be allocated automatically and returned through *to_req. Note |
2442 | * @to: ring we wish to use the object on. May be NULL. |
2722 | * that it is not guaranteed that commands will be emitted (because the system |
2443 | * |
2723 | * might already be idle). Hence there is no need to create a request that |
2444 | * This code is meant to abstract object synchronization with the GPU. |
2724 | * might never have any work submitted. Note further that if a request is |
2445 | * Calling with NULL implies synchronizing the object with the CPU |
2725 | * returned in *to_req, it is the responsibility of the caller to submit |
2446 | * rather than a particular GPU ring. |
2726 | * that request (after potentially adding more work to it). |
- | 2727 | * |
|
2447 | * |
2728 | * Returns 0 if successful, else propagates up the lower layer error. |
2448 | * Returns 0 if successful, else propagates up the lower layer error. |
2729 | */ |
2449 | */ |
2730 | int |
2450 | int |
2731 | i915_gem_object_sync(struct drm_i915_gem_object *obj, |
Line 2451... | Line 2732... | ||
2451 | i915_gem_object_sync(struct drm_i915_gem_object *obj, |
2732 | struct intel_engine_cs *to, |
2452 | struct intel_engine_cs *to) |
2733 | struct drm_i915_gem_request **to_req) |
Line 2453... | Line 2734... | ||
2453 | { |
2734 | { |
2454 | struct intel_engine_cs *from = obj->ring; |
2735 | const bool readonly = obj->base.pending_write_domain == 0; |
2455 | u32 seqno; |
- | |
2456 | int ret, idx; |
- | |
Line -... | Line 2736... | ||
- | 2736 | struct drm_i915_gem_request *req[I915_NUM_RINGS]; |
|
- | 2737 | int ret, i, n; |
|
2457 | 2738 | ||
2458 | if (from == NULL || to == from) |
2739 | if (!obj->active) |
- | 2740 | return 0; |
|
2459 | return 0; |
2741 | |
2460 | 2742 | if (to == NULL) |
|
2461 | if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) |
2743 | return i915_gem_object_wait_rendering(obj, readonly); |
2462 | return i915_gem_object_wait_rendering(obj, false); |
2744 | |
- | 2745 | n = 0; |
|
2463 | 2746 | if (readonly) { |
|
2464 | idx = intel_ring_sync_index(from, to); |
2747 | if (obj->last_write_req) |
2465 | 2748 | req[n++] = obj->last_write_req; |
|
- | 2749 | } else { |
|
Line 2466... | Line -... | ||
2466 | seqno = obj->last_read_seqno; |
- | |
2467 | /* Optimization: Avoid semaphore sync when we are sure we already |
- | |
2468 | * waited for an object with higher seqno */ |
- | |
2469 | if (seqno <= from->semaphore.sync_seqno[idx]) |
- | |
2470 | return 0; |
- | |
2471 | - | ||
2472 | ret = i915_gem_check_olr(obj->ring, seqno); |
- | |
2473 | if (ret) |
- | |
2474 | return ret; |
- | |
2475 | 2750 | for (i = 0; i < I915_NUM_RINGS; i++) |
|
2476 | trace_i915_gem_ring_sync_to(from, to, seqno); |
2751 | if (obj->last_read_req[i]) |
Line 2477... | Line 2752... | ||
2477 | ret = to->semaphore.sync_to(to, from, seqno); |
2752 | req[n++] = obj->last_read_req[i]; |
2478 | if (!ret) |
2753 | } |
2479 | /* We use last_read_seqno because sync_to() |
2754 | for (i = 0; i < n; i++) { |
Line 2480... | Line 2755... | ||
2480 | * might have just caused seqno wrap under |
2755 | ret = __i915_gem_object_sync(obj, to, req[i], to_req); |
2481 | * the radar. |
2756 | if (ret) |
Line 2482... | Line 2757... | ||
2482 | */ |
2757 | return ret; |
2483 | from->semaphore.sync_seqno[idx] = obj->last_read_seqno; |
2758 | } |
Line 2484... | Line 2759... | ||
2484 | 2759 | ||
Line 2507... | Line 2782... | ||
2507 | trace_i915_gem_object_change_domain(obj, |
2782 | trace_i915_gem_object_change_domain(obj, |
2508 | old_read_domains, |
2783 | old_read_domains, |
2509 | old_write_domain); |
2784 | old_write_domain); |
2510 | } |
2785 | } |
Line 2511... | Line 2786... | ||
2511 | 2786 | ||
2512 | int i915_vma_unbind(struct i915_vma *vma) |
2787 | static int __i915_vma_unbind(struct i915_vma *vma, bool wait) |
2513 | { |
2788 | { |
2514 | struct drm_i915_gem_object *obj = vma->obj; |
2789 | struct drm_i915_gem_object *obj = vma->obj; |
2515 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2790 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2516 | int ret; |
2791 | int ret; |
Line 2531... | Line 2806... | ||
2531 | if (vma->pin_count) |
2806 | if (vma->pin_count) |
2532 | return -EBUSY; |
2807 | return -EBUSY; |
Line 2533... | Line 2808... | ||
2533 | 2808 | ||
Line -... | Line 2809... | ||
- | 2809 | BUG_ON(obj->pages == NULL); |
|
2534 | BUG_ON(obj->pages == NULL); |
2810 | |
2535 | 2811 | if (wait) { |
|
2536 | ret = i915_gem_object_finish_gpu(obj); |
2812 | ret = i915_gem_object_wait_rendering(obj, false); |
2537 | if (ret) |
- | |
2538 | return ret; |
- | |
2539 | /* Continue on if we fail due to EIO, the GPU is hung so we |
- | |
2540 | * should be safe and we need to cleanup or else we might |
2813 | if (ret) |
2541 | * cause memory corruption through use-after-free. |
- | |
2542 | */ |
- | |
2543 | - | ||
Line 2544... | Line 2814... | ||
2544 | /* Throw away the active reference before moving to the unbound list */ |
2814 | return ret; |
- | 2815 | } |
|
2545 | i915_gem_object_retire(obj); |
2816 | |
Line 2546... | Line 2817... | ||
2546 | 2817 | if (i915_is_ggtt(vma->vm) && |
|
2547 | if (i915_is_ggtt(vma->vm)) { |
2818 | vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { |
2548 | i915_gem_object_finish_gtt(obj); |
2819 | i915_gem_object_finish_gtt(obj); |
2549 | 2820 | ||
2550 | /* release the fence reg _after_ flushing */ |
2821 | /* release the fence reg _after_ flushing */ |
Line 2551... | Line 2822... | ||
2551 | ret = i915_gem_object_put_fence(obj); |
2822 | ret = i915_gem_object_put_fence(obj); |
Line 2552... | Line 2823... | ||
2552 | if (ret) |
2823 | if (ret) |
- | 2824 | return ret; |
|
Line 2553... | Line 2825... | ||
2553 | return ret; |
2825 | } |
2554 | } |
2826 | |
- | 2827 | trace_i915_vma_unbind(vma); |
|
2555 | 2828 | ||
- | 2829 | vma->vm->unbind_vma(vma); |
|
- | 2830 | vma->bound = 0; |
|
- | 2831 | ||
- | 2832 | list_del_init(&vma->mm_list); |
|
- | 2833 | if (i915_is_ggtt(vma->vm)) { |
|
- | 2834 | if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { |
|
Line 2556... | Line 2835... | ||
2556 | trace_i915_vma_unbind(vma); |
2835 | obj->map_and_fenceable = false; |
2557 | 2836 | } else if (vma->ggtt_view.pages) { |
|
Line 2558... | Line 2837... | ||
2558 | vma->unbind_vma(vma); |
2837 | sg_free_table(vma->ggtt_view.pages); |
2559 | 2838 | kfree(vma->ggtt_view.pages); |
|
2560 | list_del_init(&vma->mm_list); |
2839 | } |
2561 | if (i915_is_ggtt(vma->vm)) |
- | |
2562 | obj->map_and_fenceable = false; |
2840 | vma->ggtt_view.pages = NULL; |
2563 | - | ||
Line 2564... | Line 2841... | ||
2564 | drm_mm_remove_node(&vma->node); |
2841 | } |
2565 | i915_gem_vma_destroy(vma); |
2842 | |
2566 | 2843 | drm_mm_remove_node(&vma->node); |
|
2567 | /* Since the unbound list is global, only move to that list if |
2844 | i915_gem_vma_destroy(vma); |
2568 | * no more VMAs exist. */ |
2845 | |
Line 2569... | Line 2846... | ||
2569 | if (list_empty(&obj->vma_list)) { |
2846 | /* Since the unbound list is global, only move to that list if |
2570 | i915_gem_gtt_finish_object(obj); |
2847 | * no more VMAs exist. */ |
Line 2571... | Line 2848... | ||
2571 | list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); |
2848 | if (list_empty(&obj->vma_list)) |
2572 | } |
- | |
2573 | - | ||
2574 | /* And finally now the object is completely decoupled from this vma, |
- | |
2575 | * we can drop its hold on the backing storage and allow it to be |
- | |
2576 | * reaped by the shrinker. |
- | |
2577 | */ |
- | |
2578 | i915_gem_object_unpin_pages(obj); |
- | |
2579 | - | ||
2580 | return 0; |
- | |
2581 | } |
- | |
2582 | - | ||
2583 | int i915_gpu_idle(struct drm_device *dev) |
- | |
2584 | { |
- | |
2585 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2586 | struct intel_engine_cs *ring; |
- | |
2587 | int ret, i; |
- | |
2588 | - | ||
2589 | /* Flush everything onto the inactive list. */ |
- | |
2590 | for_each_ring(ring, dev_priv, i) { |
- | |
2591 | if (!i915.enable_execlists) { |
- | |
2592 | ret = i915_switch_context(ring, ring->default_context); |
- | |
2593 | if (ret) |
- | |
2594 | return ret; |
- | |
2595 | } |
- | |
2596 | - | ||
2597 | ret = intel_ring_idle(ring); |
- | |
2598 | if (ret) |
- | |
2599 | return ret; |
- | |
2600 | } |
- | |
2601 | - | ||
2602 | return 0; |
- | |
2603 | } |
- | |
2604 | - | ||
2605 | static void i965_write_fence_reg(struct drm_device *dev, int reg, |
- | |
2606 | struct drm_i915_gem_object *obj) |
- | |
2607 | { |
- | |
2608 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2609 | int fence_reg; |
- | |
2610 | int fence_pitch_shift; |
- | |
2611 | - | ||
2612 | if (INTEL_INFO(dev)->gen >= 6) { |
- | |
2613 | fence_reg = FENCE_REG_SANDYBRIDGE_0; |
- | |
2614 | fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT; |
- | |
2615 | } else { |
- | |
2616 | fence_reg = FENCE_REG_965_0; |
- | |
2617 | fence_pitch_shift = I965_FENCE_PITCH_SHIFT; |
- | |
2618 | } |
- | |
2619 | - | ||
2620 | fence_reg += reg * 8; |
- | |
2621 | - | ||
2622 | /* To w/a incoherency with non-atomic 64-bit register updates, |
- | |
2623 | * we split the 64-bit update into two 32-bit writes. In order |
- | |
2624 | * for a partial fence not to be evaluated between writes, we |
- | |
2625 | * precede the update with write to turn off the fence register, |
- | |
2626 | * and only enable the fence as the last step. |
- | |
2627 | * |
- | |
2628 | * For extra levels of paranoia, we make sure each step lands |
- | |
2629 | * before applying the next step. |
- | |
2630 | */ |
- | |
2631 | I915_WRITE(fence_reg, 0); |
- | |
2632 | POSTING_READ(fence_reg); |
- | |
2633 | - | ||
2634 | if (obj) { |
- | |
2635 | u32 size = i915_gem_obj_ggtt_size(obj); |
- | |
2636 | uint64_t val; |
- | |
2637 | - | ||
2638 | val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) & |
- | |
2639 | 0xfffff000) << 32; |
- | |
2640 | val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000; |
- | |
2641 | val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; |
- | |
2642 | if (obj->tiling_mode == I915_TILING_Y) |
- | |
2643 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
- | |
2644 | val |= I965_FENCE_REG_VALID; |
- | |
2645 | - | ||
2646 | I915_WRITE(fence_reg + 4, val >> 32); |
- | |
2647 | POSTING_READ(fence_reg + 4); |
- | |
2648 | - | ||
2649 | I915_WRITE(fence_reg + 0, val); |
- | |
2650 | POSTING_READ(fence_reg); |
- | |
2651 | } else { |
- | |
2652 | I915_WRITE(fence_reg + 4, 0); |
- | |
2653 | POSTING_READ(fence_reg + 4); |
- | |
2654 | } |
- | |
2655 | } |
- | |
2656 | - | ||
2657 | static void i915_write_fence_reg(struct drm_device *dev, int reg, |
- | |
2658 | struct drm_i915_gem_object *obj) |
- | |
2659 | { |
- | |
2660 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2661 | u32 val; |
- | |
2662 | - | ||
2663 | if (obj) { |
- | |
2664 | u32 size = i915_gem_obj_ggtt_size(obj); |
- | |
2665 | int pitch_val; |
- | |
2666 | int tile_width; |
- | |
2667 | - | ||
2668 | WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) || |
- | |
2669 | (size & -size) != size || |
- | |
2670 | (i915_gem_obj_ggtt_offset(obj) & (size - 1)), |
- | |
2671 | "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", |
- | |
2672 | i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size); |
- | |
2673 | - | ||
2674 | if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) |
- | |
2675 | tile_width = 128; |
- | |
2676 | else |
- | |
2677 | tile_width = 512; |
- | |
2678 | - | ||
2679 | /* Note: pitch better be a power of two tile widths */ |
- | |
2680 | pitch_val = obj->stride / tile_width; |
- | |
2681 | pitch_val = ffs(pitch_val) - 1; |
- | |
2682 | - | ||
2683 | val = i915_gem_obj_ggtt_offset(obj); |
- | |
2684 | if (obj->tiling_mode == I915_TILING_Y) |
- | |
2685 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
- | |
2686 | val |= I915_FENCE_SIZE_BITS(size); |
- | |
2687 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
- | |
2688 | val |= I830_FENCE_REG_VALID; |
- | |
2689 | } else |
- | |
2690 | val = 0; |
- | |
2691 | 2849 | list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); |
|
2692 | if (reg < 8) |
- | |
2693 | reg = FENCE_REG_830_0 + reg * 4; |
- | |
2694 | else |
- | |
2695 | reg = FENCE_REG_945_8 + (reg - 8) * 4; |
- | |
2696 | - | ||
2697 | I915_WRITE(reg, val); |
- | |
2698 | POSTING_READ(reg); |
- | |
2699 | } |
- | |
2700 | - | ||
2701 | static void i830_write_fence_reg(struct drm_device *dev, int reg, |
- | |
2702 | struct drm_i915_gem_object *obj) |
- | |
2703 | { |
- | |
2704 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2705 | uint32_t val; |
- | |
2706 | - | ||
2707 | if (obj) { |
- | |
2708 | u32 size = i915_gem_obj_ggtt_size(obj); |
2850 | |
2709 | uint32_t pitch_val; |
- | |
2710 | - | ||
2711 | WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) || |
- | |
2712 | (size & -size) != size || |
- | |
2713 | (i915_gem_obj_ggtt_offset(obj) & (size - 1)), |
- | |
2714 | "object 0x%08lx not 512K or pot-size 0x%08x aligned\n", |
- | |
2715 | i915_gem_obj_ggtt_offset(obj), size); |
- | |
2716 | - | ||
2717 | pitch_val = obj->stride / 128; |
- | |
2718 | pitch_val = ffs(pitch_val) - 1; |
- | |
2719 | 2851 | /* And finally now the object is completely decoupled from this vma, |
|
Line 2720... | Line 2852... | ||
2720 | val = i915_gem_obj_ggtt_offset(obj); |
2852 | * we can drop its hold on the backing storage and allow it to be |
2721 | if (obj->tiling_mode == I915_TILING_Y) |
2853 | * reaped by the shrinker. |
2722 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
2854 | */ |
2723 | val |= I830_FENCE_SIZE_BITS(size); |
2855 | i915_gem_object_unpin_pages(obj); |
Line 2724... | Line 2856... | ||
2724 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
2856 | |
2725 | val |= I830_FENCE_REG_VALID; |
- | |
2726 | } else |
2857 | return 0; |
2727 | val = 0; |
2858 | } |
- | 2859 | ||
- | 2860 | int i915_vma_unbind(struct i915_vma *vma) |
|
Line 2728... | Line -... | ||
2728 | - | ||
2729 | I915_WRITE(FENCE_REG_830_0 + reg * 4, val); |
- | |
2730 | POSTING_READ(FENCE_REG_830_0 + reg * 4); |
- | |
2731 | } |
- | |
2732 | - | ||
2733 | inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj) |
- | |
2734 | { |
- | |
2735 | return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT; |
- | |
2736 | } |
- | |
2737 | - | ||
2738 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
- | |
2739 | struct drm_i915_gem_object *obj) |
- | |
2740 | { |
- | |
2741 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2742 | - | ||
2743 | /* Ensure that all CPU reads are completed before installing a fence |
- | |
2744 | * and all writes before removing the fence. |
- | |
2745 | */ |
- | |
2746 | if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) |
- | |
2747 | mb(); |
- | |
2748 | - | ||
2749 | WARN(obj && (!obj->stride || !obj->tiling_mode), |
- | |
2750 | "bogus fence setup with stride: 0x%x, tiling mode: %i\n", |
- | |
2751 | obj->stride, obj->tiling_mode); |
2861 | { |
2752 | - | ||
2753 | switch (INTEL_INFO(dev)->gen) { |
- | |
2754 | case 9: |
- | |
2755 | case 8: |
- | |
2756 | case 7: |
- | |
2757 | case 6: |
- | |
2758 | case 5: |
- | |
2759 | case 4: i965_write_fence_reg(dev, reg, obj); break; |
- | |
2760 | case 3: i915_write_fence_reg(dev, reg, obj); break; |
- | |
2761 | case 2: i830_write_fence_reg(dev, reg, obj); break; |
- | |
2762 | default: BUG(); |
- | |
2763 | } |
- | |
2764 | - | ||
2765 | /* And similarly be paranoid that no direct access to this region |
- | |
2766 | * is reordered to before the fence is installed. |
- | |
2767 | */ |
- | |
2768 | if (i915_gem_object_needs_mb(obj)) |
2862 | return __i915_vma_unbind(vma, true); |
2769 | mb(); |
- | |
2770 | } |
- | |
2771 | - | ||
2772 | static inline int fence_number(struct drm_i915_private *dev_priv, |
- | |
2773 | struct drm_i915_fence_reg *fence) |
- | |
2774 | { |
- | |
2775 | return fence - dev_priv->fence_regs; |
- | |
2776 | } |
- | |
2777 | - | ||
2778 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
- | |
2779 | struct drm_i915_fence_reg *fence, |
- | |
2780 | bool enable) |
- | |
2781 | { |
- | |
2782 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
- | |
2783 | int reg = fence_number(dev_priv, fence); |
- | |
2784 | - | ||
2785 | i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); |
- | |
2786 | - | ||
2787 | if (enable) { |
2863 | } |
2788 | obj->fence_reg = reg; |
- | |
2789 | fence->obj = obj; |
- | |
2790 | list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
- | |
2791 | } else { |
- | |
2792 | obj->fence_reg = I915_FENCE_REG_NONE; |
- | |
2793 | fence->obj = NULL; |
- | |
2794 | list_del_init(&fence->lru_list); |
- | |
2795 | } |
- | |
2796 | obj->fence_dirty = false; |
- | |
2797 | } |
- | |
2798 | - | ||
2799 | static int |
- | |
2800 | i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) |
- | |
2801 | { |
- | |
2802 | if (obj->last_fenced_seqno) { |
2864 | |
2803 | int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); |
- | |
Line 2804... | Line 2865... | ||
2804 | if (ret) |
2865 | int __i915_vma_unbind_no_wait(struct i915_vma *vma) |
2805 | return ret; |
2866 | { |
2806 | 2867 | return __i915_vma_unbind(vma, false); |
|
Line 2807... | Line -... | ||
2807 | obj->last_fenced_seqno = 0; |
- | |
2808 | } |
- | |
2809 | - | ||
2810 | return 0; |
- | |
2811 | } |
- | |
2812 | - | ||
2813 | int |
- | |
2814 | i915_gem_object_put_fence(struct drm_i915_gem_object *obj) |
- | |
2815 | { |
- | |
2816 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
- | |
2817 | struct drm_i915_fence_reg *fence; |
- | |
2818 | int ret; |
- | |
2819 | - | ||
2820 | ret = i915_gem_object_wait_fence(obj); |
- | |
2821 | if (ret) |
- | |
2822 | return ret; |
- | |
2823 | - | ||
2824 | if (obj->fence_reg == I915_FENCE_REG_NONE) |
- | |
2825 | return 0; |
- | |
2826 | - | ||
2827 | fence = &dev_priv->fence_regs[obj->fence_reg]; |
- | |
2828 | - | ||
2829 | if (WARN_ON(fence->pin_count)) |
- | |
2830 | return -EBUSY; |
- | |
2831 | 2868 | } |
|
2832 | i915_gem_object_fence_lost(obj); |
2869 | |
2833 | i915_gem_object_update_fence(obj, fence, false); |
- | |
2834 | - | ||
2835 | return 0; |
- | |
2836 | } |
- | |
2837 | - | ||
2838 | static struct drm_i915_fence_reg * |
- | |
2839 | i915_find_fence_reg(struct drm_device *dev) |
- | |
2840 | { |
- | |
2841 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2842 | struct drm_i915_fence_reg *reg, *avail; |
- | |
2843 | int i; |
- | |
2844 | - | ||
2845 | /* First try to find a free reg */ |
- | |
2846 | avail = NULL; |
- | |
2847 | for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { |
- | |
2848 | reg = &dev_priv->fence_regs[i]; |
- | |
2849 | if (!reg->obj) |
- | |
2850 | return reg; |
- | |
2851 | - | ||
2852 | if (!reg->pin_count) |
- | |
2853 | avail = reg; |
- | |
2854 | } |
- | |
2855 | - | ||
2856 | if (avail == NULL) |
- | |
2857 | goto deadlock; |
- | |
2858 | - | ||
2859 | /* None available, try to steal one or wait for a user to finish */ |
- | |
2860 | list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { |
- | |
2861 | if (reg->pin_count) |
- | |
2862 | continue; |
- | |
2863 | - | ||
2864 | return reg; |
- | |
2865 | } |
- | |
2866 | - | ||
2867 | deadlock: |
- | |
2868 | /* Wait for completion of pending flips which consume fences */ |
- | |
2869 | // if (intel_has_pending_fb_unpin(dev)) |
- | |
2870 | // return ERR_PTR(-EAGAIN); |
- | |
2871 | - | ||
2872 | return ERR_PTR(-EDEADLK); |
- | |
2873 | } |
- | |
2874 | - | ||
2875 | /** |
- | |
2876 | * i915_gem_object_get_fence - set up fencing for an object |
- | |
2877 | * @obj: object to map through a fence reg |
- | |
2878 | * |
- | |
2879 | * When mapping objects through the GTT, userspace wants to be able to write |
- | |
2880 | * to them without having to worry about swizzling if the object is tiled. |
- | |
2881 | * This function walks the fence regs looking for a free one for @obj, |
- | |
2882 | * stealing one if it can't find any. |
- | |
2883 | * |
- | |
2884 | * It then sets up the reg based on the object's properties: address, pitch |
- | |
2885 | * and tiling format. |
2870 | int i915_gpu_idle(struct drm_device *dev) |
2886 | * |
- | |
2887 | * For an untiled surface, this removes any existing fence. |
2871 | { |
2888 | */ |
2872 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 2889... | Line -... | ||
2889 | int |
- | |
2890 | i915_gem_object_get_fence(struct drm_i915_gem_object *obj) |
- | |
2891 | { |
- | |
2892 | struct drm_device *dev = obj->base.dev; |
- | |
2893 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
2894 | bool enable = obj->tiling_mode != I915_TILING_NONE; |
2873 | struct intel_engine_cs *ring; |
2895 | struct drm_i915_fence_reg *reg; |
- | |
2896 | int ret; |
2874 | int ret, i; |
2897 | - | ||
2898 | /* Have we updated the tiling parameters upon the object and so |
- | |
2899 | * will need to serialise the write to the associated fence register? |
- | |
2900 | */ |
- | |
2901 | if (obj->fence_dirty) { |
- | |
2902 | ret = i915_gem_object_wait_fence(obj); |
- | |
2903 | if (ret) |
- | |
2904 | return ret; |
- | |
2905 | } |
- | |
2906 | - | ||
Line 2907... | Line 2875... | ||
2907 | /* Just update our place in the LRU if our fence is getting reused. */ |
2875 | |
2908 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
2876 | /* Flush everything onto the inactive list. */ |
2909 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
2877 | for_each_ring(ring, dev_priv, i) { |
2910 | if (!obj->fence_dirty) { |
- | |
2911 | list_move_tail(®->lru_list, |
- | |
2912 | &dev_priv->mm.fence_list); |
2878 | if (!i915.enable_execlists) { |
2913 | return 0; |
- | |
2914 | } |
- | |
2915 | } else if (enable) { |
- | |
2916 | if (WARN_ON(!obj->map_and_fenceable)) |
- | |
Line -... | Line 2879... | ||
- | 2879 | struct drm_i915_gem_request *req; |
|
2917 | return -EINVAL; |
2880 | |
2918 | 2881 | ret = i915_gem_request_alloc(ring, ring->default_context, &req); |
|
Line 2919... | Line 2882... | ||
2919 | reg = i915_find_fence_reg(dev); |
2882 | if (ret) |
2920 | if (IS_ERR(reg)) |
2883 | return ret; |
Line 2969... | Line 2932... | ||
2969 | 2932 | ||
2970 | return true; |
2933 | return true; |
Line 2971... | Line 2934... | ||
2971 | } |
2934 | } |
2972 | 2935 | ||
- | 2936 | /** |
|
2973 | /** |
2937 | * Finds free space in the GTT aperture and binds the object or a view of it |
2974 | * Finds free space in the GTT aperture and binds the object there. |
2938 | * there. |
2975 | */ |
2939 | */ |
2976 | static struct i915_vma * |
2940 | static struct i915_vma * |
- | 2941 | i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, |
|
2977 | i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, |
2942 | struct i915_address_space *vm, |
2978 | struct i915_address_space *vm, |
2943 | const struct i915_ggtt_view *ggtt_view, |
2979 | unsigned alignment, |
2944 | unsigned alignment, |
2980 | uint64_t flags) |
2945 | uint64_t flags) |
2981 | { |
2946 | { |
2982 | struct drm_device *dev = obj->base.dev; |
2947 | struct drm_device *dev = obj->base.dev; |
2983 | struct drm_i915_private *dev_priv = dev->dev_private; |
2948 | struct drm_i915_private *dev_priv = dev->dev_private; |
2984 | u32 size, fence_size, fence_alignment, unfenced_alignment; |
- | |
2985 | unsigned long start = |
2949 | u32 fence_alignment, unfenced_alignment; |
2986 | flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; |
2950 | u32 search_flag, alloc_flag; |
2987 | unsigned long end = |
2951 | u64 start, end; |
2988 | flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total; |
2952 | u64 size, fence_size; |
Line -... | Line 2953... | ||
- | 2953 | struct i915_vma *vma; |
|
- | 2954 | int ret; |
|
- | 2955 | ||
- | 2956 | if (i915_is_ggtt(vm)) { |
|
- | 2957 | u32 view_size; |
|
- | 2958 | ||
- | 2959 | if (WARN_ON(!ggtt_view)) |
|
- | 2960 | return ERR_PTR(-EINVAL); |
|
- | 2961 | ||
- | 2962 | view_size = i915_ggtt_view_size(obj, ggtt_view); |
|
- | 2963 | ||
- | 2964 | fence_size = i915_gem_get_gtt_size(dev, |
|
- | 2965 | view_size, |
|
- | 2966 | obj->tiling_mode); |
|
- | 2967 | fence_alignment = i915_gem_get_gtt_alignment(dev, |
|
- | 2968 | view_size, |
|
- | 2969 | obj->tiling_mode, |
|
- | 2970 | true); |
|
- | 2971 | unfenced_alignment = i915_gem_get_gtt_alignment(dev, |
|
- | 2972 | view_size, |
|
- | 2973 | obj->tiling_mode, |
|
2989 | struct i915_vma *vma; |
2974 | false); |
2990 | int ret; |
2975 | size = flags & PIN_MAPPABLE ? fence_size : view_size; |
2991 | 2976 | } else { |
|
2992 | fence_size = i915_gem_get_gtt_size(dev, |
2977 | fence_size = i915_gem_get_gtt_size(dev, |
2993 | obj->base.size, |
2978 | obj->base.size, |
2994 | obj->tiling_mode); |
2979 | obj->tiling_mode); |
- | 2980 | fence_alignment = i915_gem_get_gtt_alignment(dev, |
|
2995 | fence_alignment = i915_gem_get_gtt_alignment(dev, |
2981 | obj->base.size, |
2996 | obj->base.size, |
2982 | obj->tiling_mode, |
2997 | obj->tiling_mode, true); |
2983 | true); |
2998 | unfenced_alignment = |
2984 | unfenced_alignment = |
- | 2985 | i915_gem_get_gtt_alignment(dev, |
|
- | 2986 | obj->base.size, |
|
- | 2987 | obj->tiling_mode, |
|
- | 2988 | false); |
|
- | 2989 | size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; |
|
- | 2990 | } |
|
- | 2991 | ||
- | 2992 | start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; |
|
- | 2993 | end = vm->total; |
|
- | 2994 | if (flags & PIN_MAPPABLE) |
|
Line 2999... | Line 2995... | ||
2999 | i915_gem_get_gtt_alignment(dev, |
2995 | end = min_t(u64, end, dev_priv->gtt.mappable_end); |
3000 | obj->base.size, |
2996 | if (flags & PIN_ZONE_4G) |
3001 | obj->tiling_mode, false); |
2997 | end = min_t(u64, end, (1ULL << 32)); |
3002 | 2998 | ||
3003 | if (alignment == 0) |
2999 | if (alignment == 0) |
- | 3000 | alignment = flags & PIN_MAPPABLE ? fence_alignment : |
|
- | 3001 | unfenced_alignment; |
|
3004 | alignment = flags & PIN_MAPPABLE ? fence_alignment : |
3002 | if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) { |
3005 | unfenced_alignment; |
3003 | DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n", |
Line 3006... | Line 3004... | ||
3006 | if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) { |
3004 | ggtt_view ? ggtt_view->type : 0, |
3007 | DRM_DEBUG("Invalid object alignment requested %u\n", alignment); |
- | |
3008 | return ERR_PTR(-EINVAL); |
3005 | alignment); |
3009 | } |
3006 | return ERR_PTR(-EINVAL); |
3010 | 3007 | } |
|
3011 | size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; |
3008 | |
3012 | 3009 | /* If binding the object/GGTT view requires more space than the entire |
|
- | 3010 | * aperture has, reject it early before evicting everything in a vain |
|
3013 | /* If the object is bigger than the entire aperture, reject it early |
3011 | * attempt to find space. |
3014 | * before evicting everything in a vain attempt to find space. |
3012 | */ |
3015 | */ |
3013 | if (size > end) { |
3016 | if (obj->base.size > end) { |
3014 | DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n", |
3017 | DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n", |
3015 | ggtt_view ? ggtt_view->type : 0, |
Line 3025... | Line 3023... | ||
3025 | if (ret) |
3023 | if (ret) |
3026 | return ERR_PTR(ret); |
3024 | return ERR_PTR(ret); |
Line 3027... | Line 3025... | ||
3027 | 3025 | ||
Line -... | Line 3026... | ||
- | 3026 | i915_gem_object_pin_pages(obj); |
|
3028 | i915_gem_object_pin_pages(obj); |
3027 | |
- | 3028 | vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) : |
|
3029 | 3029 | i915_gem_obj_lookup_or_create_vma(obj, vm); |
|
3030 | vma = i915_gem_obj_lookup_or_create_vma(obj, vm); |
3030 | |
Line -... | Line 3031... | ||
- | 3031 | if (IS_ERR(vma)) |
|
- | 3032 | goto err_unpin; |
|
- | 3033 | ||
- | 3034 | if (flags & PIN_HIGH) { |
|
- | 3035 | search_flag = DRM_MM_SEARCH_BELOW; |
|
- | 3036 | alloc_flag = DRM_MM_CREATE_TOP; |
|
- | 3037 | } else { |
|
- | 3038 | search_flag = DRM_MM_SEARCH_DEFAULT; |
|
3031 | if (IS_ERR(vma)) |
3039 | alloc_flag = DRM_MM_CREATE_DEFAULT; |
3032 | goto err_unpin; |
3040 | } |
3033 | 3041 | ||
3034 | search_free: |
3042 | search_free: |
3035 | ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, |
3043 | ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, |
3036 | size, alignment, |
3044 | size, alignment, |
3037 | obj->cache_level, |
3045 | obj->cache_level, |
3038 | start, end, |
3046 | start, end, |
Line 3039... | Line 3047... | ||
3039 | DRM_MM_SEARCH_DEFAULT, |
3047 | search_flag, |
3040 | DRM_MM_CREATE_DEFAULT); |
3048 | alloc_flag); |
3041 | if (ret) { |
3049 | if (ret) { |
3042 | 3050 | ||
3043 | goto err_free_vma; |
3051 | goto err_free_vma; |
3044 | } |
3052 | } |
Line -... | Line 3053... | ||
- | 3053 | if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { |
|
3045 | if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { |
3054 | ret = -EINVAL; |
3046 | ret = -EINVAL; |
3055 | goto err_remove_node; |
3047 | goto err_remove_node; |
3056 | } |
Line 3048... | Line 3057... | ||
3048 | } |
3057 | |
3049 | 3058 | trace_i915_vma_bind(vma, flags); |
|
Line 3050... | Line -... | ||
3050 | ret = i915_gem_gtt_prepare_object(obj); |
- | |
3051 | if (ret) |
- | |
3052 | goto err_remove_node; |
- | |
3053 | - | ||
3054 | list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); |
3059 | ret = i915_vma_bind(vma, obj->cache_level, flags); |
Line 3055... | Line 3060... | ||
3055 | list_add_tail(&vma->mm_list, &vm->inactive_list); |
3060 | if (ret) |
3056 | 3061 | goto err_remove_node; |
|
3057 | trace_i915_vma_bind(vma, flags); |
3062 | |
Line 3094... | Line 3099... | ||
3094 | * flushed/invalidated. As we always have to emit invalidations |
3099 | * flushed/invalidated. As we always have to emit invalidations |
3095 | * and flushes when moving into and out of the RENDER domain, correct |
3100 | * and flushes when moving into and out of the RENDER domain, correct |
3096 | * snooping behaviour occurs naturally as the result of our domain |
3101 | * snooping behaviour occurs naturally as the result of our domain |
3097 | * tracking. |
3102 | * tracking. |
3098 | */ |
3103 | */ |
3099 | if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) |
3104 | if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { |
- | 3105 | obj->cache_dirty = true; |
|
3100 | return false; |
3106 | return false; |
- | 3107 | } |
|
Line 3101... | Line 3108... | ||
3101 | 3108 | ||
3102 | trace_i915_gem_object_clflush(obj); |
3109 | trace_i915_gem_object_clflush(obj); |
- | 3110 | drm_clflush_sg(obj->pages); |
|
Line 3103... | Line 3111... | ||
3103 | drm_clflush_sg(obj->pages); |
3111 | obj->cache_dirty = false; |
3104 | 3112 | ||
Line 3105... | Line 3113... | ||
3105 | return true; |
3113 | return true; |
Line 3125... | Line 3133... | ||
3125 | wmb(); |
3133 | wmb(); |
Line 3126... | Line 3134... | ||
3126 | 3134 | ||
3127 | old_write_domain = obj->base.write_domain; |
3135 | old_write_domain = obj->base.write_domain; |
Line 3128... | Line 3136... | ||
3128 | obj->base.write_domain = 0; |
3136 | obj->base.write_domain = 0; |
Line 3129... | Line 3137... | ||
3129 | 3137 | ||
3130 | intel_fb_obj_flush(obj, false); |
3138 | intel_fb_obj_flush(obj, false, ORIGIN_GTT); |
3131 | 3139 | ||
3132 | trace_i915_gem_object_change_domain(obj, |
3140 | trace_i915_gem_object_change_domain(obj, |
Line 3133... | Line 3141... | ||
3133 | obj->base.read_domains, |
3141 | obj->base.read_domains, |
3134 | old_write_domain); |
3142 | old_write_domain); |
3135 | } |
3143 | } |
3136 | - | ||
3137 | /** Flushes the CPU write domain for the object if it's dirty. */ |
3144 | |
3138 | static void |
3145 | /** Flushes the CPU write domain for the object if it's dirty. */ |
Line 3139... | Line 3146... | ||
3139 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, |
3146 | static void |
3140 | bool force) |
3147 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) |
Line 3141... | Line 3148... | ||
3141 | { |
3148 | { |
3142 | uint32_t old_write_domain; |
3149 | uint32_t old_write_domain; |
Line 3143... | Line 3150... | ||
3143 | 3150 | ||
3144 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
3151 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
Line 3145... | Line 3152... | ||
3145 | return; |
3152 | return; |
Line 3146... | Line 3153... | ||
3146 | 3153 | ||
3147 | if (i915_gem_clflush_object(obj, force)) |
3154 | if (i915_gem_clflush_object(obj, obj->pin_display)) |
3148 | i915_gem_chipset_flush(obj->base.dev); |
3155 | i915_gem_chipset_flush(obj->base.dev); |
3149 | 3156 | ||
Line 3164... | Line 3171... | ||
3164 | * flushes to occur. |
3171 | * flushes to occur. |
3165 | */ |
3172 | */ |
3166 | int |
3173 | int |
3167 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
3174 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
3168 | { |
3175 | { |
3169 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
- | |
3170 | struct i915_vma *vma = i915_gem_obj_to_ggtt(obj); |
- | |
3171 | uint32_t old_write_domain, old_read_domains; |
3176 | uint32_t old_write_domain, old_read_domains; |
- | 3177 | struct i915_vma *vma; |
|
3172 | int ret; |
3178 | int ret; |
Line 3173... | Line -... | ||
3173 | - | ||
3174 | /* Not valid to be called on unbound objects. */ |
- | |
3175 | if (vma == NULL) |
- | |
3176 | return -EINVAL; |
- | |
3177 | 3179 | ||
3178 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
3180 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
Line 3179... | Line 3181... | ||
3179 | return 0; |
3181 | return 0; |
3180 | 3182 | ||
3181 | ret = i915_gem_object_wait_rendering(obj, !write); |
3183 | ret = i915_gem_object_wait_rendering(obj, !write); |
Line -... | Line 3184... | ||
- | 3184 | if (ret) |
|
- | 3185 | return ret; |
|
- | 3186 | ||
- | 3187 | /* Flush and acquire obj->pages so that we are coherent through |
|
- | 3188 | * direct access in memory with previous cached writes through |
|
- | 3189 | * shmemfs and that our cache domain tracking remains valid. |
|
- | 3190 | * For example, if the obj->filp was moved to swap without us |
|
- | 3191 | * being notified and releasing the pages, we would mistakenly |
|
3182 | if (ret) |
3192 | * continue to assume that the obj remained out of the CPU cached |
- | 3193 | * domain. |
|
- | 3194 | */ |
|
- | 3195 | ret = i915_gem_object_get_pages(obj); |
|
3183 | return ret; |
3196 | if (ret) |
Line 3184... | Line 3197... | ||
3184 | 3197 | return ret; |
|
3185 | i915_gem_object_retire(obj); |
3198 | |
3186 | i915_gem_object_flush_cpu_write_domain(obj, false); |
3199 | i915_gem_object_flush_cpu_write_domain(obj); |
3187 | 3200 | ||
Line 3204... | Line 3217... | ||
3204 | obj->base.read_domains = I915_GEM_DOMAIN_GTT; |
3217 | obj->base.read_domains = I915_GEM_DOMAIN_GTT; |
3205 | obj->base.write_domain = I915_GEM_DOMAIN_GTT; |
3218 | obj->base.write_domain = I915_GEM_DOMAIN_GTT; |
3206 | obj->dirty = 1; |
3219 | obj->dirty = 1; |
3207 | } |
3220 | } |
Line 3208... | Line -... | ||
3208 | - | ||
3209 | if (write) |
- | |
3210 | intel_fb_obj_invalidate(obj, NULL); |
- | |
3211 | 3221 | ||
3212 | trace_i915_gem_object_change_domain(obj, |
3222 | trace_i915_gem_object_change_domain(obj, |
3213 | old_read_domains, |
3223 | old_read_domains, |
Line 3214... | Line 3224... | ||
3214 | old_write_domain); |
3224 | old_write_domain); |
3215 | 3225 | ||
- | 3226 | /* And bump the LRU for this access */ |
|
3216 | /* And bump the LRU for this access */ |
3227 | vma = i915_gem_obj_to_ggtt(obj); |
3217 | if (i915_gem_object_is_inactive(obj)) |
3228 | if (vma && drm_mm_node_allocated(&vma->node) && !obj->active) |
Line 3218... | Line 3229... | ||
3218 | list_move_tail(&vma->mm_list, |
3229 | list_move_tail(&vma->mm_list, |
3219 | &dev_priv->gtt.base.inactive_list); |
3230 | &to_i915(obj->base.dev)->gtt.base.inactive_list); |
Line -... | Line 3231... | ||
- | 3231 | ||
- | 3232 | return 0; |
|
- | 3233 | } |
|
- | 3234 | ||
- | 3235 | /** |
|
- | 3236 | * Changes the cache-level of an object across all VMA. |
|
- | 3237 | * |
|
- | 3238 | * After this function returns, the object will be in the new cache-level |
|
- | 3239 | * across all GTT and the contents of the backing storage will be coherent, |
|
- | 3240 | * with respect to the new cache-level. In order to keep the backing storage |
|
- | 3241 | * coherent for all users, we only allow a single cache level to be set |
|
- | 3242 | * globally on the object and prevent it from being changed whilst the |
|
- | 3243 | * hardware is reading from the object. That is if the object is currently |
|
3220 | 3244 | * on the scanout it will be set to uncached (or equivalent display |
|
3221 | return 0; |
3245 | * cache coherency) and all non-MOCS GPU access will also be uncached so |
3222 | } |
3246 | * that all direct access to the scanout remains coherent. |
3223 | 3247 | */ |
|
3224 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
3248 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
- | 3249 | enum i915_cache_level cache_level) |
|
3225 | enum i915_cache_level cache_level) |
3250 | { |
Line 3226... | Line 3251... | ||
3226 | { |
3251 | struct drm_device *dev = obj->base.dev; |
3227 | struct drm_device *dev = obj->base.dev; |
3252 | struct i915_vma *vma, *next; |
Line -... | Line 3253... | ||
- | 3253 | bool bound = false; |
|
- | 3254 | int ret = 0; |
|
- | 3255 | ||
- | 3256 | if (obj->cache_level == cache_level) |
|
- | 3257 | goto out; |
|
- | 3258 | ||
3228 | struct i915_vma *vma, *next; |
3259 | /* Inspect the list of currently bound VMA and unbind any that would |
- | 3260 | * be invalid given the new cache-level. This is principally to |
|
- | 3261 | * catch the issue of the CS prefetch crossing page boundaries and |
|
- | 3262 | * reading an invalid PTE on older architectures. |
|
3229 | int ret; |
3263 | */ |
3230 | 3264 | list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { |
|
3231 | if (obj->cache_level == cache_level) |
3265 | if (!drm_mm_node_allocated(&vma->node)) |
Line 3232... | Line -... | ||
3232 | return 0; |
- | |
3233 | 3266 | continue; |
|
3234 | if (i915_gem_obj_is_pinned(obj)) { |
3267 | |
3235 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
3268 | if (vma->pin_count) { |
3236 | return -EBUSY; |
3269 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
3237 | } |
3270 | return -EBUSY; |
- | 3271 | } |
|
3238 | 3272 | ||
Line -... | Line 3273... | ||
- | 3273 | if (!i915_gem_valid_gtt_space(vma, cache_level)) { |
|
- | 3274 | ret = i915_vma_unbind(vma); |
|
- | 3275 | if (ret) |
|
- | 3276 | return ret; |
|
- | 3277 | } else |
|
- | 3278 | bound = true; |
|
- | 3279 | } |
|
3239 | list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { |
3280 | |
- | 3281 | /* We can reuse the existing drm_mm nodes but need to change the |
|
- | 3282 | * cache-level on the PTE. We could simply unbind them all and |
|
- | 3283 | * rebind with the correct cache-level on next use. However since |
|
- | 3284 | * we already have a valid slot, dma mapping, pages etc, we may as |
|
3240 | if (!i915_gem_valid_gtt_space(vma, cache_level)) { |
3285 | * rewrite the PTE in the belief that doing so tramples upon less |
3241 | ret = i915_vma_unbind(vma); |
3286 | * state and so involves less work. |
3242 | if (ret) |
3287 | */ |
Line -... | Line 3288... | ||
- | 3288 | if (bound) { |
|
- | 3289 | /* Before we change the PTE, the GPU must not be accessing it. |
|
- | 3290 | * If we wait upon the object, we know that all the bound |
|
- | 3291 | * VMA are no longer active. |
|
- | 3292 | */ |
|
- | 3293 | ret = i915_gem_object_wait_rendering(obj, false); |
|
- | 3294 | if (ret) |
|
- | 3295 | return ret; |
|
3243 | return ret; |
3296 | |
Line -... | Line 3297... | ||
- | 3297 | if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) { |
|
3244 | } |
3298 | /* Access to snoopable pages through the GTT is |
3245 | } |
3299 | * incoherent and on some machines causes a hard |
- | 3300 | * lockup. Relinquish the CPU mmaping to force |
|
3246 | 3301 | * userspace to refault in the pages and we can |
|
- | 3302 | * then double check if the GTT mapping is still |
|
3247 | if (i915_gem_obj_bound_any(obj)) { |
3303 | * valid for that pointer access. |
3248 | ret = i915_gem_object_finish_gpu(obj); |
- | |
3249 | if (ret) |
3304 | */ |
3250 | return ret; |
3305 | i915_gem_release_mmap(obj); |
3251 | 3306 | ||
- | 3307 | /* As we no longer need a fence for GTT access, |
|
- | 3308 | * we can relinquish it now (and so prevent having |
|
- | 3309 | * to steal a fence from someone else on the next |
|
- | 3310 | * fence request). Note GPU activity would have |
|
- | 3311 | * dropped the fence as all snoopable access is |
|
- | 3312 | * supposed to be linear. |
|
- | 3313 | */ |
|
- | 3314 | ret = i915_gem_object_put_fence(obj); |
|
3252 | i915_gem_object_finish_gtt(obj); |
3315 | if (ret) |
Line 3253... | Line 3316... | ||
3253 | 3316 | return ret; |
|
3254 | /* Before SandyBridge, you could not use tiling or fence |
3317 | } else { |
- | 3318 | /* We either have incoherent backing store and |
|
- | 3319 | * so no GTT access or the architecture is fully |
|
3255 | * registers with snooped memory, so relinquish any fences |
3320 | * coherent. In such cases, existing GTT mmaps |
- | 3321 | * ignore the cache bit in the PTE and we can |
|
3256 | * currently pointing to our region in the aperture. |
3322 | * rewrite it without confusing the GPU or having |
- | 3323 | * to force userspace to fault back in its mmaps. |
|
3257 | */ |
3324 | */ |
Line 3258... | Line 3325... | ||
3258 | if (INTEL_INFO(dev)->gen < 6) { |
3325 | } |
3259 | ret = i915_gem_object_put_fence(obj); |
3326 | |
3260 | if (ret) |
3327 | list_for_each_entry(vma, &obj->vma_list, vma_link) { |
Line 3261... | Line -... | ||
3261 | return ret; |
- | |
3262 | } |
- | |
3263 | 3328 | if (!drm_mm_node_allocated(&vma->node)) |
|
3264 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
- | |
3265 | if (drm_mm_node_allocated(&vma->node)) |
3329 | continue; |
3266 | vma->bind_vma(vma, cache_level, |
3330 | |
3267 | vma->bound & GLOBAL_BIND); |
- | |
3268 | } |
3331 | ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); |
3269 | 3332 | if (ret) |
|
3270 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
3333 | return ret; |
3271 | vma->node.color = cache_level; |
- | |
3272 | obj->cache_level = cache_level; |
- | |
3273 | - | ||
3274 | if (cpu_write_needs_clflush(obj)) { |
- | |
3275 | u32 old_read_domains, old_write_domain; |
- | |
3276 | 3334 | } |
|
3277 | /* If we're coming from LLC cached, then we haven't |
3335 | } |
3278 | * actually been tracking whether the data is in the |
- | |
3279 | * CPU cache or not, since we only allow one bit set |
3336 | |
3280 | * in obj->write_domain and have been skipping the clflushes. |
- | |
3281 | * Just set it to the CPU cache for now. |
3337 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
3282 | */ |
3338 | vma->node.color = cache_level; |
Line 3283... | Line 3339... | ||
3283 | i915_gem_object_retire(obj); |
3339 | obj->cache_level = cache_level; |
3284 | WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); |
3340 | |
Line 3285... | Line 3341... | ||
3285 | 3341 | out: |
|
3286 | old_read_domains = obj->base.read_domains; |
3342 | /* Flush the dirty CPU caches to the backing storage so that the |
3287 | old_write_domain = obj->base.write_domain; |
3343 | * object is now coherent at its new cache level (with respect |
3288 | 3344 | * to the access domain). |
|
3289 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3345 | */ |
3290 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
- | |
3291 | - | ||
3292 | trace_i915_gem_object_change_domain(obj, |
- | |
3293 | old_read_domains, |
- | |
3294 | old_write_domain); |
- | |
Line 3295... | Line 3346... | ||
3295 | } |
3346 | if (obj->cache_dirty && |
3296 | 3347 | obj->base.write_domain != I915_GEM_DOMAIN_CPU && |
|
3297 | return 0; |
3348 | cpu_write_needs_clflush(obj)) { |
3298 | } |
- | |
3299 | - | ||
Line 3300... | Line 3349... | ||
3300 | int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, |
3349 | if (i915_gem_clflush_object(obj, true)) |
3301 | struct drm_file *file) |
3350 | i915_gem_chipset_flush(obj->base.dev); |
3302 | { |
3351 | } |
3303 | struct drm_i915_gem_caching *args = data; |
3352 | |
Line 3327... | Line 3376... | ||
3327 | default: |
3376 | default: |
3328 | args->caching = I915_CACHING_NONE; |
3377 | args->caching = I915_CACHING_NONE; |
3329 | break; |
3378 | break; |
3330 | } |
3379 | } |
Line 3331... | Line 3380... | ||
3331 | 3380 | ||
3332 | drm_gem_object_unreference(&obj->base); |
- | |
3333 | unlock: |
- | |
3334 | mutex_unlock(&dev->struct_mutex); |
3381 | drm_gem_object_unreference_unlocked(&obj->base); |
3335 | return ret; |
3382 | return 0; |
Line 3336... | Line 3383... | ||
3336 | } |
3383 | } |
3337 | 3384 | ||
3338 | int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, |
3385 | int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, |
- | 3386 | struct drm_file *file) |
|
3339 | struct drm_file *file) |
3387 | { |
3340 | { |
3388 | struct drm_i915_private *dev_priv = dev->dev_private; |
3341 | struct drm_i915_gem_caching *args = data; |
3389 | struct drm_i915_gem_caching *args = data; |
3342 | struct drm_i915_gem_object *obj; |
3390 | struct drm_i915_gem_object *obj; |
Line 3343... | Line 3391... | ||
3343 | enum i915_cache_level level; |
3391 | enum i915_cache_level level; |
3344 | int ret; |
3392 | int ret; |
3345 | 3393 | ||
3346 | switch (args->caching) { |
3394 | switch (args->caching) { |
3347 | case I915_CACHING_NONE: |
3395 | case I915_CACHING_NONE: |
- | 3396 | level = I915_CACHE_NONE; |
|
- | 3397 | break; |
|
- | 3398 | case I915_CACHING_CACHED: |
|
- | 3399 | /* |
|
- | 3400 | * Due to a HW issue on BXT A stepping, GPU stores via a |
|
- | 3401 | * snooped mapping may leave stale data in a corresponding CPU |
|
- | 3402 | * cacheline, whereas normally such cachelines would get |
|
- | 3403 | * invalidated. |
|
- | 3404 | */ |
|
3348 | level = I915_CACHE_NONE; |
3405 | if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) |
3349 | break; |
3406 | return -ENODEV; |
3350 | case I915_CACHING_CACHED: |
3407 | |
3351 | level = I915_CACHE_LLC; |
3408 | level = I915_CACHE_LLC; |
3352 | break; |
3409 | break; |
3353 | case I915_CACHING_DISPLAY: |
3410 | case I915_CACHING_DISPLAY: |
3354 | level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE; |
3411 | level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE; |
3355 | break; |
3412 | break; |
Line -... | Line 3413... | ||
- | 3413 | default: |
|
- | 3414 | return -EINVAL; |
|
3356 | default: |
3415 | } |
3357 | return -EINVAL; |
3416 | |
3358 | } |
3417 | intel_runtime_pm_get(dev_priv); |
Line 3359... | Line 3418... | ||
3359 | 3418 | ||
3360 | ret = i915_mutex_lock_interruptible(dev); |
3419 | ret = i915_mutex_lock_interruptible(dev); |
3361 | if (ret) |
3420 | if (ret) |
3362 | return ret; |
3421 | goto rpm_put; |
Line 3370... | Line 3429... | ||
3370 | ret = i915_gem_object_set_cache_level(obj, level); |
3429 | ret = i915_gem_object_set_cache_level(obj, level); |
Line 3371... | Line 3430... | ||
3371 | 3430 | ||
3372 | drm_gem_object_unreference(&obj->base); |
3431 | drm_gem_object_unreference(&obj->base); |
3373 | unlock: |
3432 | unlock: |
3374 | mutex_unlock(&dev->struct_mutex); |
3433 | mutex_unlock(&dev->struct_mutex); |
3375 | return ret; |
- | |
3376 | } |
- | |
3377 | - | ||
3378 | static bool is_pin_display(struct drm_i915_gem_object *obj) |
- | |
3379 | { |
- | |
3380 | struct i915_vma *vma; |
- | |
3381 | 3434 | rpm_put: |
|
3382 | vma = i915_gem_obj_to_ggtt(obj); |
- | |
3383 | if (!vma) |
- | |
Line 3384... | Line -... | ||
3384 | return false; |
- | |
3385 | - | ||
3386 | /* There are 3 sources that pin objects: |
- | |
3387 | * 1. The display engine (scanouts, sprites, cursors); |
3435 | intel_runtime_pm_put(dev_priv); |
3388 | * 2. Reservations for execbuffer; |
- | |
3389 | * 3. The user. |
- | |
3390 | * |
- | |
3391 | * We can ignore reservations as we hold the struct_mutex and |
- | |
3392 | * are only called outside of the reservation path. The user |
- | |
3393 | * can only increment pin_count once, and so if after |
- | |
3394 | * subtracting the potential reference by the user, any pin_count |
- | |
3395 | * remains, it must be due to another use by the display engine. |
- | |
3396 | */ |
3436 | |
Line 3397... | Line 3437... | ||
3397 | return vma->pin_count - !!obj->user_pin_count; |
3437 | return ret; |
3398 | } |
3438 | } |
3399 | 3439 | ||
3400 | /* |
3440 | /* |
3401 | * Prepare buffer for display plane (scanout, cursors, etc). |
3441 | * Prepare buffer for display plane (scanout, cursors, etc). |
3402 | * Can be called from an uninterruptible phase (modesetting) and allows |
3442 | * Can be called from an uninterruptible phase (modesetting) and allows |
3403 | * any flushes to be pipelined (for pageflips). |
3443 | * any flushes to be pipelined (for pageflips). |
3404 | */ |
3444 | */ |
3405 | int |
3445 | int |
- | 3446 | i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, |
|
- | 3447 | u32 alignment, |
|
3406 | i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, |
3448 | struct intel_engine_cs *pipelined, |
3407 | u32 alignment, |
3449 | struct drm_i915_gem_request **pipelined_request, |
3408 | struct intel_engine_cs *pipelined) |
- | |
3409 | { |
3450 | const struct i915_ggtt_view *view) |
Line 3410... | Line -... | ||
3410 | u32 old_read_domains, old_write_domain; |
- | |
3411 | bool was_pin_display; |
3451 | { |
3412 | int ret; |
3452 | u32 old_read_domains, old_write_domain; |
3413 | 3453 | int ret; |
|
3414 | if (pipelined != obj->ring) { |
- | |
Line 3415... | Line 3454... | ||
3415 | ret = i915_gem_object_sync(obj, pipelined); |
3454 | |
3416 | if (ret) |
3455 | ret = i915_gem_object_sync(obj, pipelined, pipelined_request); |
3417 | return ret; |
3456 | if (ret) |
3418 | } |
- | |
3419 | 3457 | return ret; |
|
Line 3420... | Line 3458... | ||
3420 | /* Mark the pin_display early so that we account for the |
3458 | |
3421 | * display coherency whilst setting up the cache domains. |
3459 | /* Mark the pin_display early so that we account for the |
3422 | */ |
3460 | * display coherency whilst setting up the cache domains. |
3423 | was_pin_display = obj->pin_display; |
3461 | */ |
Line 3439... | Line 3477... | ||
3439 | 3477 | ||
3440 | /* As the user may map the buffer once pinned in the display plane |
3478 | /* As the user may map the buffer once pinned in the display plane |
3441 | * (e.g. libkms for the bootup splash), we have to ensure that we |
3479 | * (e.g. libkms for the bootup splash), we have to ensure that we |
3442 | * always use map_and_fenceable for all scanout buffers. |
3480 | * always use map_and_fenceable for all scanout buffers. |
3443 | */ |
3481 | */ |
- | 3482 | ret = i915_gem_object_ggtt_pin(obj, view, alignment, |
|
- | 3483 | view->type == I915_GGTT_VIEW_NORMAL ? |
|
3444 | ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE); |
3484 | PIN_MAPPABLE : 0); |
3445 | if (ret) |
3485 | if (ret) |
Line 3446... | Line 3486... | ||
3446 | goto err_unpin_display; |
3486 | goto err_unpin_display; |
Line 3447... | Line 3487... | ||
3447 | 3487 | ||
3448 | i915_gem_object_flush_cpu_write_domain(obj, true); |
3488 | i915_gem_object_flush_cpu_write_domain(obj); |
Line 3449... | Line 3489... | ||
3449 | 3489 | ||
Line 3461... | Line 3501... | ||
3461 | old_write_domain); |
3501 | old_write_domain); |
Line 3462... | Line 3502... | ||
3462 | 3502 | ||
Line 3463... | Line 3503... | ||
3463 | return 0; |
3503 | return 0; |
3464 | - | ||
3465 | err_unpin_display: |
3504 | |
3466 | WARN_ON(was_pin_display != is_pin_display(obj)); |
3505 | err_unpin_display: |
3467 | obj->pin_display = was_pin_display; |
3506 | obj->pin_display--; |
Line 3468... | Line 3507... | ||
3468 | return ret; |
3507 | return ret; |
3469 | } |
3508 | } |
- | 3509 | ||
3470 | 3510 | void |
|
3471 | void |
- | |
3472 | i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) |
3511 | i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, |
3473 | { |
- | |
3474 | i915_gem_object_ggtt_unpin(obj); |
- | |
3475 | obj->pin_display = is_pin_display(obj); |
- | |
3476 | } |
- | |
3477 | - | ||
3478 | int |
- | |
3479 | i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) |
- | |
3480 | { |
- | |
3481 | int ret; |
3512 | const struct i915_ggtt_view *view) |
Line 3482... | Line 3513... | ||
3482 | 3513 | { |
|
3483 | if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) |
- | |
3484 | return 0; |
- | |
Line 3485... | Line -... | ||
3485 | - | ||
3486 | ret = i915_gem_object_wait_rendering(obj, false); |
- | |
3487 | if (ret) |
3514 | if (WARN_ON(obj->pin_display == 0)) |
3488 | return ret; |
3515 | return; |
Line 3489... | Line 3516... | ||
3489 | 3516 | ||
3490 | /* Ensure that we invalidate the GPU's caches and TLBs. */ |
3517 | i915_gem_object_ggtt_unpin_view(obj, view); |
3491 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
3518 | |
Line 3509... | Line 3536... | ||
3509 | 3536 | ||
3510 | ret = i915_gem_object_wait_rendering(obj, !write); |
3537 | ret = i915_gem_object_wait_rendering(obj, !write); |
3511 | if (ret) |
3538 | if (ret) |
Line 3512... | Line -... | ||
3512 | return ret; |
- | |
3513 | 3539 | return ret; |
|
Line 3514... | Line 3540... | ||
3514 | i915_gem_object_retire(obj); |
3540 | |
3515 | i915_gem_object_flush_gtt_write_domain(obj); |
3541 | i915_gem_object_flush_gtt_write_domain(obj); |
Line 3535... | Line 3561... | ||
3535 | if (write) { |
3561 | if (write) { |
3536 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3562 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3537 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3563 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3538 | } |
3564 | } |
Line 3539... | Line -... | ||
3539 | - | ||
3540 | if (write) |
- | |
3541 | intel_fb_obj_invalidate(obj, NULL); |
- | |
3542 | 3565 | ||
3543 | trace_i915_gem_object_change_domain(obj, |
3566 | trace_i915_gem_object_change_domain(obj, |
3544 | old_read_domains, |
3567 | old_read_domains, |
Line 3545... | Line 3568... | ||
3545 | old_write_domain); |
3568 | old_write_domain); |
Line 3560... | Line 3583... | ||
3560 | static int |
3583 | static int |
3561 | i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) |
3584 | i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) |
3562 | { |
3585 | { |
3563 | struct drm_i915_private *dev_priv = dev->dev_private; |
3586 | struct drm_i915_private *dev_priv = dev->dev_private; |
3564 | struct drm_i915_file_private *file_priv = file->driver_priv; |
3587 | struct drm_i915_file_private *file_priv = file->driver_priv; |
3565 | unsigned long recent_enough = jiffies - msecs_to_jiffies(20); |
3588 | unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; |
3566 | struct drm_i915_gem_request *request; |
3589 | struct drm_i915_gem_request *request, *target = NULL; |
3567 | struct intel_engine_cs *ring = NULL; |
- | |
3568 | unsigned reset_counter; |
3590 | unsigned reset_counter; |
3569 | u32 seqno = 0; |
- | |
3570 | int ret; |
3591 | int ret; |
Line 3571... | Line 3592... | ||
3571 | 3592 | ||
3572 | ret = i915_gem_wait_for_error(&dev_priv->gpu_error); |
3593 | ret = i915_gem_wait_for_error(&dev_priv->gpu_error); |
3573 | if (ret) |
3594 | if (ret) |
Line 3580... | Line 3601... | ||
3580 | spin_lock(&file_priv->mm.lock); |
3601 | spin_lock(&file_priv->mm.lock); |
3581 | list_for_each_entry(request, &file_priv->mm.request_list, client_list) { |
3602 | list_for_each_entry(request, &file_priv->mm.request_list, client_list) { |
3582 | if (time_after_eq(request->emitted_jiffies, recent_enough)) |
3603 | if (time_after_eq(request->emitted_jiffies, recent_enough)) |
3583 | break; |
3604 | break; |
Line -... | Line 3605... | ||
- | 3605 | ||
- | 3606 | /* |
|
- | 3607 | * Note that the request might not have been submitted yet. |
|
- | 3608 | * In which case emitted_jiffies will be zero. |
|
3584 | 3609 | */ |
|
- | 3610 | if (!request->emitted_jiffies) |
|
- | 3611 | continue; |
|
3585 | ring = request->ring; |
3612 | |
3586 | seqno = request->seqno; |
3613 | target = request; |
3587 | } |
3614 | } |
- | 3615 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
|
- | 3616 | if (target) |
|
3588 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
3617 | i915_gem_request_reference(target); |
Line 3589... | Line 3618... | ||
3589 | spin_unlock(&file_priv->mm.lock); |
3618 | spin_unlock(&file_priv->mm.lock); |
3590 | 3619 | ||
Line 3591... | Line 3620... | ||
3591 | if (seqno == 0) |
3620 | if (target == NULL) |
3592 | return 0; |
3621 | return 0; |
3593 | 3622 | ||
Line -... | Line 3623... | ||
- | 3623 | ret = __i915_wait_request(target, reset_counter, true, NULL, NULL); |
|
- | 3624 | if (ret == 0) |
|
3594 | ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL); |
3625 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); |
3595 | if (ret == 0) |
3626 | |
Line 3596... | Line 3627... | ||
3596 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); |
3627 | i915_gem_request_unreference__unlocked(target); |
3597 | 3628 | ||
Line 3615... | Line 3646... | ||
3615 | return true; |
3646 | return true; |
Line 3616... | Line 3647... | ||
3616 | 3647 | ||
3617 | return false; |
3648 | return false; |
Line -... | Line 3649... | ||
- | 3649 | } |
|
3618 | } |
3650 | |
- | 3651 | void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) |
|
- | 3652 | { |
|
- | 3653 | struct drm_i915_gem_object *obj = vma->obj; |
|
- | 3654 | bool mappable, fenceable; |
|
- | 3655 | u32 fence_size, fence_alignment; |
|
- | 3656 | ||
- | 3657 | fence_size = i915_gem_get_gtt_size(obj->base.dev, |
|
- | 3658 | obj->base.size, |
|
- | 3659 | obj->tiling_mode); |
|
- | 3660 | fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev, |
|
- | 3661 | obj->base.size, |
|
- | 3662 | obj->tiling_mode, |
|
- | 3663 | true); |
|
- | 3664 | ||
- | 3665 | fenceable = (vma->node.size == fence_size && |
|
- | 3666 | (vma->node.start & (fence_alignment - 1)) == 0); |
|
- | 3667 | ||
- | 3668 | mappable = (vma->node.start + fence_size <= |
|
- | 3669 | to_i915(obj->base.dev)->gtt.mappable_end); |
|
- | 3670 | ||
- | 3671 | obj->map_and_fenceable = mappable && fenceable; |
|
- | 3672 | } |
|
3619 | 3673 | ||
3620 | int |
3674 | static int |
- | 3675 | i915_gem_object_do_pin(struct drm_i915_gem_object *obj, |
|
3621 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
3676 | struct i915_address_space *vm, |
3622 | struct i915_address_space *vm, |
3677 | const struct i915_ggtt_view *ggtt_view, |
3623 | uint32_t alignment, |
3678 | uint32_t alignment, |
3624 | uint64_t flags) |
3679 | uint64_t flags) |
3625 | { |
3680 | { |
Line 3635... | Line 3690... | ||
3635 | return -EINVAL; |
3690 | return -EINVAL; |
Line 3636... | Line 3691... | ||
3636 | 3691 | ||
3637 | if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE)) |
3692 | if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE)) |
Line -... | Line 3693... | ||
- | 3693 | return -EINVAL; |
|
- | 3694 | ||
- | 3695 | if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) |
|
- | 3696 | return -EINVAL; |
|
3638 | return -EINVAL; |
3697 | |
- | 3698 | vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) : |
|
- | 3699 | i915_gem_obj_to_vma(obj, vm); |
|
- | 3700 | ||
- | 3701 | if (IS_ERR(vma)) |
|
3639 | 3702 | return PTR_ERR(vma); |
|
3640 | vma = i915_gem_obj_to_vma(obj, vm); |
3703 | |
3641 | if (vma) { |
3704 | if (vma) { |
Line 3642... | Line 3705... | ||
3642 | if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) |
3705 | if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) |
3643 | return -EBUSY; |
3706 | return -EBUSY; |
3644 | 3707 | ||
3645 | if (i915_vma_misplaced(vma, alignment, flags)) { |
3708 | if (i915_vma_misplaced(vma, alignment, flags)) { |
3646 | WARN(vma->pin_count, |
3709 | WARN(vma->pin_count, |
- | 3710 | "bo is already pinned in %s with incorrect alignment:" |
|
- | 3711 | " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d," |
|
3647 | "bo is already pinned with incorrect alignment:" |
3712 | " obj->map_and_fenceable=%d\n", |
- | 3713 | ggtt_view ? "ggtt" : "ppgtt", |
|
3648 | " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d," |
3714 | upper_32_bits(vma->node.start), |
3649 | " obj->map_and_fenceable=%d\n", |
3715 | lower_32_bits(vma->node.start), |
3650 | i915_gem_obj_offset(obj, vm), alignment, |
3716 | alignment, |
3651 | !!(flags & PIN_MAPPABLE), |
3717 | !!(flags & PIN_MAPPABLE), |
3652 | obj->map_and_fenceable); |
3718 | obj->map_and_fenceable); |
Line 3658... | Line 3724... | ||
3658 | } |
3724 | } |
3659 | } |
3725 | } |
Line 3660... | Line 3726... | ||
3660 | 3726 | ||
3661 | bound = vma ? vma->bound : 0; |
3727 | bound = vma ? vma->bound : 0; |
3662 | if (vma == NULL || !drm_mm_node_allocated(&vma->node)) { |
3728 | if (vma == NULL || !drm_mm_node_allocated(&vma->node)) { |
- | 3729 | vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment, |
|
3663 | vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags); |
3730 | flags); |
3664 | if (IS_ERR(vma)) |
3731 | if (IS_ERR(vma)) |
- | 3732 | return PTR_ERR(vma); |
|
- | 3733 | } else { |
|
- | 3734 | ret = i915_vma_bind(vma, obj->cache_level, flags); |
|
- | 3735 | if (ret) |
|
3665 | return PTR_ERR(vma); |
3736 | return ret; |
Line 3666... | Line 3737... | ||
3666 | } |
3737 | } |
3667 | - | ||
3668 | if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND)) |
- | |
3669 | vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND); |
3738 | |
3670 | 3739 | if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL && |
|
3671 | if ((bound ^ vma->bound) & GLOBAL_BIND) { |
- | |
3672 | bool mappable, fenceable; |
- | |
3673 | u32 fence_size, fence_alignment; |
- | |
3674 | - | ||
3675 | fence_size = i915_gem_get_gtt_size(obj->base.dev, |
- | |
3676 | obj->base.size, |
- | |
3677 | obj->tiling_mode); |
- | |
3678 | fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev, |
- | |
3679 | obj->base.size, |
- | |
3680 | obj->tiling_mode, |
- | |
3681 | true); |
- | |
3682 | - | ||
3683 | fenceable = (vma->node.size == fence_size && |
- | |
3684 | (vma->node.start & (fence_alignment - 1)) == 0); |
- | |
3685 | - | ||
3686 | mappable = (vma->node.start + obj->base.size <= |
- | |
3687 | dev_priv->gtt.mappable_end); |
- | |
3688 | - | ||
3689 | obj->map_and_fenceable = mappable && fenceable; |
- | |
3690 | } |
3740 | (bound ^ vma->bound) & GLOBAL_BIND) { |
- | 3741 | __i915_vma_set_map_and_fenceable(vma); |
|
Line 3691... | Line 3742... | ||
3691 | 3742 | WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); |
|
3692 | WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); |
- | |
3693 | - | ||
3694 | vma->pin_count++; |
- | |
3695 | if (flags & PIN_MAPPABLE) |
3743 | } |
3696 | obj->pin_mappable |= true; |
3744 | |
Line 3697... | Line 3745... | ||
3697 | 3745 | vma->pin_count++; |
|
3698 | return 0; |
3746 | return 0; |
3699 | } |
- | |
3700 | 3747 | } |
|
3701 | void |
- | |
3702 | i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj) |
- | |
3703 | { |
- | |
3704 | struct i915_vma *vma = i915_gem_obj_to_ggtt(obj); |
- | |
3705 | - | ||
3706 | BUG_ON(!vma); |
3748 | |
3707 | BUG_ON(vma->pin_count == 0); |
3749 | int |
3708 | BUG_ON(!i915_gem_obj_ggtt_bound(obj)); |
- | |
3709 | - | ||
3710 | if (--vma->pin_count == 0) |
- | |
3711 | obj->pin_mappable = false; |
- | |
3712 | } |
3750 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
3713 | - | ||
3714 | bool |
3751 | struct i915_address_space *vm, |
3715 | i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) |
3752 | uint32_t alignment, |
3716 | { |
- | |
3717 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
- | |
3718 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
- | |
3719 | struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj); |
3753 | uint64_t flags) |
3720 | - | ||
3721 | WARN_ON(!ggtt_vma || |
- | |
3722 | dev_priv->fence_regs[obj->fence_reg].pin_count > |
- | |
3723 | ggtt_vma->pin_count); |
- | |
3724 | dev_priv->fence_regs[obj->fence_reg].pin_count++; |
- | |
3725 | return true; |
- | |
3726 | } else |
- | |
3727 | return false; |
- | |
3728 | } |
- | |
3729 | - | ||
3730 | void |
- | |
3731 | i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) |
- | |
3732 | { |
- | |
3733 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
- | |
3734 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
3754 | { |
Line 3735... | Line 3755... | ||
3735 | WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); |
3755 | return i915_gem_object_do_pin(obj, vm, |
3736 | dev_priv->fence_regs[obj->fence_reg].pin_count--; |
3756 | i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL, |
3737 | } |
3757 | alignment, flags); |
- | 3758 | } |
|
- | 3759 | ||
3738 | } |
3760 | int |
3739 | - | ||
3740 | int |
- | |
3741 | i915_gem_pin_ioctl(struct drm_device *dev, void *data, |
- | |
3742 | struct drm_file *file) |
- | |
3743 | { |
- | |
3744 | struct drm_i915_gem_pin *args = data; |
- | |
3745 | struct drm_i915_gem_object *obj; |
- | |
3746 | int ret; |
- | |
3747 | - | ||
3748 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
- | |
3749 | return -ENODEV; |
- | |
3750 | - | ||
3751 | ret = i915_mutex_lock_interruptible(dev); |
- | |
3752 | if (ret) |
- | |
3753 | return ret; |
- | |
3754 | - | ||
3755 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
- | |
3756 | if (&obj->base == NULL) { |
- | |
3757 | ret = -ENOENT; |
- | |
3758 | goto unlock; |
- | |
3759 | } |
- | |
3760 | - | ||
3761 | if (obj->madv != I915_MADV_WILLNEED) { |
- | |
3762 | DRM_DEBUG("Attempting to pin a purgeable buffer\n"); |
3761 | i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, |
3763 | ret = -EFAULT; |
- | |
3764 | goto out; |
- | |
3765 | } |
3762 | const struct i915_ggtt_view *view, |
3766 | - | ||
3767 | if (obj->pin_filp != NULL && obj->pin_filp != file) { |
- | |
3768 | DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n", |
- | |
3769 | args->handle); |
- | |
3770 | ret = -EINVAL; |
- | |
3771 | goto out; |
- | |
3772 | } |
- | |
Line 3773... | Line -... | ||
3773 | - | ||
3774 | if (obj->user_pin_count == ULONG_MAX) { |
3763 | uint32_t alignment, |
3775 | ret = -EBUSY; |
- | |
3776 | goto out; |
3764 | uint64_t flags) |
3777 | } |
3765 | { |
Line 3778... | Line -... | ||
3778 | - | ||
3779 | if (obj->user_pin_count == 0) { |
- | |
3780 | ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE); |
- | |
3781 | if (ret) |
- | |
3782 | goto out; |
3766 | if (WARN_ONCE(!view, "no view specified")) |
3783 | } |
- | |
3784 | - | ||
3785 | obj->user_pin_count++; |
- | |
3786 | obj->pin_filp = file; |
- | |
3787 | - | ||
3788 | args->offset = i915_gem_obj_ggtt_offset(obj); |
- | |
3789 | out: |
- | |
3790 | drm_gem_object_unreference(&obj->base); |
3767 | return -EINVAL; |
3791 | unlock: |
3768 | |
3792 | mutex_unlock(&dev->struct_mutex); |
3769 | return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view, |
3793 | return ret; |
- | |
3794 | } |
3770 | alignment, flags | PIN_GLOBAL); |
3795 | - | ||
Line 3796... | Line -... | ||
3796 | int |
- | |
3797 | i915_gem_unpin_ioctl(struct drm_device *dev, void *data, |
3771 | } |
3798 | struct drm_file *file) |
- | |
3799 | { |
3772 | |
3800 | struct drm_i915_gem_pin *args = data; |
- | |
3801 | struct drm_i915_gem_object *obj; |
- | |
3802 | int ret; |
- | |
3803 | 3773 | void |
|
3804 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
- | |
3805 | return -ENODEV; |
- | |
3806 | - | ||
3807 | ret = i915_mutex_lock_interruptible(dev); |
- | |
Line 3808... | Line -... | ||
3808 | if (ret) |
- | |
3809 | return ret; |
- | |
3810 | - | ||
3811 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
- | |
3812 | if (&obj->base == NULL) { |
- | |
3813 | ret = -ENOENT; |
- | |
3814 | goto unlock; |
3774 | i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, |
3815 | } |
- | |
3816 | - | ||
3817 | if (obj->pin_filp != file) { |
- | |
3818 | DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", |
- | |
3819 | args->handle); |
- | |
3820 | ret = -EINVAL; |
- | |
3821 | goto out; |
- | |
3822 | } |
- | |
3823 | obj->user_pin_count--; |
- | |
3824 | if (obj->user_pin_count == 0) { |
- | |
3825 | obj->pin_filp = NULL; |
3775 | const struct i915_ggtt_view *view) |
Line 3826... | Line 3776... | ||
3826 | i915_gem_object_ggtt_unpin(obj); |
3776 | { |
3827 | } |
3777 | struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view); |
3828 | 3778 | ||
Line 3855... | Line 3805... | ||
3855 | * by the gpu. Users of this interface expect objects to eventually |
3805 | * by the gpu. Users of this interface expect objects to eventually |
3856 | * become non-busy without any further actions, therefore emit any |
3806 | * become non-busy without any further actions, therefore emit any |
3857 | * necessary flushes here. |
3807 | * necessary flushes here. |
3858 | */ |
3808 | */ |
3859 | ret = i915_gem_object_flush_active(obj); |
3809 | ret = i915_gem_object_flush_active(obj); |
- | 3810 | if (ret) |
|
- | 3811 | goto unref; |
|
Line 3860... | Line -... | ||
3860 | - | ||
3861 | args->busy = obj->active; |
- | |
3862 | if (obj->ring) { |
3812 | |
3863 | BUILD_BUG_ON(I915_NUM_RINGS > 16); |
3813 | BUILD_BUG_ON(I915_NUM_RINGS > 16); |
3864 | args->busy |= intel_ring_flag(obj->ring) << 16; |
3814 | args->busy = obj->active << 16; |
- | 3815 | if (obj->last_write_req) |
|
Line -... | Line 3816... | ||
- | 3816 | args->busy |= obj->last_write_req->ring->id; |
|
3865 | } |
3817 | |
3866 | 3818 | unref: |
|
3867 | drm_gem_object_unreference(&obj->base); |
3819 | drm_gem_object_unreference(&obj->base); |
3868 | unlock: |
3820 | unlock: |
3869 | mutex_unlock(&dev->struct_mutex); |
3821 | mutex_unlock(&dev->struct_mutex); |
Line 3922... | Line 3874... | ||
3922 | 3874 | ||
3923 | if (obj->madv != __I915_MADV_PURGED) |
3875 | if (obj->madv != __I915_MADV_PURGED) |
Line 3924... | Line 3876... | ||
3924 | obj->madv = args->madv; |
3876 | obj->madv = args->madv; |
3925 | 3877 | ||
3926 | /* if the object is no longer attached, discard its backing storage */ |
3878 | /* if the object is no longer attached, discard its backing storage */ |
Line 3927... | Line 3879... | ||
3927 | if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) |
3879 | if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) |
Line 3928... | Line 3880... | ||
3928 | i915_gem_object_truncate(obj); |
3880 | i915_gem_object_truncate(obj); |
Line 3938... | Line 3890... | ||
3938 | #endif |
3890 | #endif |
Line 3939... | Line 3891... | ||
3939 | 3891 | ||
3940 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
3892 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
3941 | const struct drm_i915_gem_object_ops *ops) |
3893 | const struct drm_i915_gem_object_ops *ops) |
- | 3894 | { |
|
- | 3895 | int i; |
|
3942 | { |
3896 | |
- | 3897 | INIT_LIST_HEAD(&obj->global_list); |
|
3943 | INIT_LIST_HEAD(&obj->global_list); |
3898 | for (i = 0; i < I915_NUM_RINGS; i++) |
3944 | INIT_LIST_HEAD(&obj->ring_list); |
3899 | INIT_LIST_HEAD(&obj->ring_list[i]); |
3945 | INIT_LIST_HEAD(&obj->obj_exec_link); |
3900 | INIT_LIST_HEAD(&obj->obj_exec_link); |
- | 3901 | INIT_LIST_HEAD(&obj->vma_list); |
|
Line 3946... | Line 3902... | ||
3946 | INIT_LIST_HEAD(&obj->vma_list); |
3902 | INIT_LIST_HEAD(&obj->batch_pool_link); |
Line 3947... | Line 3903... | ||
3947 | 3903 | ||
3948 | obj->ops = ops; |
3904 | obj->ops = ops; |
Line 4067... | Line 4023... | ||
4067 | 4023 | ||
4068 | struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, |
4024 | struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, |
4069 | struct i915_address_space *vm) |
4025 | struct i915_address_space *vm) |
4070 | { |
4026 | { |
4071 | struct i915_vma *vma; |
4027 | struct i915_vma *vma; |
- | 4028 | list_for_each_entry(vma, &obj->vma_list, vma_link) { |
|
- | 4029 | if (i915_is_ggtt(vma->vm) && |
|
- | 4030 | vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) |
|
4072 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
4031 | continue; |
4073 | if (vma->vm == vm) |
4032 | if (vma->vm == vm) |
- | 4033 | return vma; |
|
- | 4034 | } |
|
- | 4035 | return NULL; |
|
- | 4036 | } |
|
- | 4037 | ||
- | 4038 | struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj, |
|
- | 4039 | const struct i915_ggtt_view *view) |
|
- | 4040 | { |
|
- | 4041 | struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); |
|
- | 4042 | struct i915_vma *vma; |
|
- | 4043 | ||
- | 4044 | if (WARN_ONCE(!view, "no view specified")) |
|
Line -... | Line 4045... | ||
- | 4045 | return ERR_PTR(-EINVAL); |
|
- | 4046 | ||
- | 4047 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
|
- | 4048 | if (vma->vm == ggtt && |
|
4074 | return vma; |
4049 | i915_ggtt_view_equal(&vma->ggtt_view, view)) |
4075 | 4050 | return vma; |
|
Line 4076... | Line 4051... | ||
4076 | return NULL; |
4051 | return NULL; |
4077 | } |
4052 | } |
Line 4093... | Line 4068... | ||
4093 | list_del(&vma->vma_link); |
4068 | list_del(&vma->vma_link); |
Line 4094... | Line 4069... | ||
4094 | 4069 | ||
4095 | kfree(vma); |
4070 | kfree(vma); |
Line -... | Line 4071... | ||
- | 4071 | } |
|
- | 4072 | ||
- | 4073 | static void |
|
- | 4074 | i915_gem_stop_ringbuffers(struct drm_device *dev) |
|
- | 4075 | { |
|
- | 4076 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 4077 | struct intel_engine_cs *ring; |
|
- | 4078 | int i; |
|
- | 4079 | ||
- | 4080 | for_each_ring(ring, dev_priv, i) |
|
- | 4081 | dev_priv->gt.stop_ring(ring); |
|
4096 | } |
4082 | } |
4097 | 4083 | ||
4098 | #if 0 |
4084 | #if 0 |
4099 | int |
4085 | int |
4100 | i915_gem_suspend(struct drm_device *dev) |
4086 | i915_gem_suspend(struct drm_device *dev) |
Line 4107... | Line 4093... | ||
4107 | if (ret) |
4093 | if (ret) |
4108 | goto err; |
4094 | goto err; |
Line 4109... | Line 4095... | ||
4109 | 4095 | ||
Line 4110... | Line -... | ||
4110 | i915_gem_retire_requests(dev); |
- | |
4111 | - | ||
4112 | /* Under UMS, be paranoid and evict. */ |
- | |
4113 | if (!drm_core_check_feature(dev, DRIVER_MODESET)) |
- | |
4114 | i915_gem_evict_everything(dev); |
4096 | i915_gem_retire_requests(dev); |
4115 | 4097 | ||
Line 4116... | Line 4098... | ||
4116 | i915_gem_stop_ringbuffers(dev); |
4098 | i915_gem_stop_ringbuffers(dev); |
4117 | mutex_unlock(&dev->struct_mutex); |
4099 | mutex_unlock(&dev->struct_mutex); |
4118 | 4100 | ||
Line -... | Line 4101... | ||
- | 4101 | cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); |
|
- | 4102 | cancel_delayed_work_sync(&dev_priv->mm.retire_work); |
|
- | 4103 | flush_delayed_work(&dev_priv->mm.idle_work); |
|
- | 4104 | ||
- | 4105 | /* Assert that we sucessfully flushed all the work and |
|
4119 | del_timer_sync(&dev_priv->gpu_error.hangcheck_timer); |
4106 | * reset the GPU back to its idle, low power state. |
Line 4120... | Line 4107... | ||
4120 | cancel_delayed_work_sync(&dev_priv->mm.retire_work); |
4107 | */ |
4121 | flush_delayed_work(&dev_priv->mm.idle_work); |
4108 | WARN_ON(dev_priv->mm.busy); |
4122 | 4109 | ||
4123 | return 0; |
4110 | return 0; |
4124 | 4111 | ||
Line 4125... | Line 4112... | ||
4125 | err: |
4112 | err: |
4126 | mutex_unlock(&dev->struct_mutex); |
4113 | mutex_unlock(&dev->struct_mutex); |
- | 4114 | return ret; |
|
4127 | return ret; |
4115 | } |
4128 | } |
4116 | #endif |
4129 | #endif |
4117 | |
4130 | 4118 | int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice) |
|
4131 | int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice) |
4119 | { |
Line 4132... | Line 4120... | ||
4132 | { |
4120 | struct intel_engine_cs *ring = req->ring; |
4133 | struct drm_device *dev = ring->dev; |
4121 | struct drm_device *dev = ring->dev; |
Line 4134... | Line 4122... | ||
4134 | struct drm_i915_private *dev_priv = dev->dev_private; |
4122 | struct drm_i915_private *dev_priv = dev->dev_private; |
4135 | u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); |
4123 | u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); |
4136 | u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; |
4124 | u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; |
Line 4137... | Line 4125... | ||
4137 | int i, ret; |
4125 | int i, ret; |
4138 | 4126 | ||
Line 4182... | Line 4170... | ||
4182 | I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); |
4170 | I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); |
4183 | else |
4171 | else |
4184 | BUG(); |
4172 | BUG(); |
4185 | } |
4173 | } |
Line 4186... | Line -... | ||
4186 | - | ||
4187 | static bool |
- | |
4188 | intel_enable_blt(struct drm_device *dev) |
- | |
4189 | { |
- | |
4190 | if (!HAS_BLT(dev)) |
- | |
4191 | return false; |
- | |
4192 | - | ||
4193 | /* The blitter was dysfunctional on early prototypes */ |
- | |
4194 | if (IS_GEN6(dev) && dev->pdev->revision < 8) { |
- | |
4195 | DRM_INFO("BLT not supported on this pre-production hardware;" |
- | |
4196 | " graphics performance will be degraded.\n"); |
- | |
4197 | return false; |
- | |
4198 | } |
- | |
4199 | - | ||
4200 | return true; |
- | |
4201 | } |
- | |
4202 | 4174 | ||
4203 | static void init_unused_ring(struct drm_device *dev, u32 base) |
4175 | static void init_unused_ring(struct drm_device *dev, u32 base) |
4204 | { |
4176 | { |
Line 4205... | Line 4177... | ||
4205 | struct drm_i915_private *dev_priv = dev->dev_private; |
4177 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 4230... | Line 4202... | ||
4230 | int i915_gem_init_rings(struct drm_device *dev) |
4202 | int i915_gem_init_rings(struct drm_device *dev) |
4231 | { |
4203 | { |
4232 | struct drm_i915_private *dev_priv = dev->dev_private; |
4204 | struct drm_i915_private *dev_priv = dev->dev_private; |
4233 | int ret; |
4205 | int ret; |
Line 4234... | Line -... | ||
4234 | - | ||
4235 | /* |
- | |
4236 | * At least 830 can leave some of the unused rings |
- | |
4237 | * "active" (ie. head != tail) after resume which |
- | |
4238 | * will prevent c3 entry. Makes sure all unused rings |
- | |
4239 | * are totally idle. |
- | |
4240 | */ |
- | |
4241 | init_unused_rings(dev); |
- | |
4242 | 4206 | ||
4243 | ret = intel_init_render_ring_buffer(dev); |
4207 | ret = intel_init_render_ring_buffer(dev); |
4244 | if (ret) |
4208 | if (ret) |
Line 4245... | Line 4209... | ||
4245 | return ret; |
4209 | return ret; |
4246 | 4210 | ||
4247 | if (HAS_BSD(dev)) { |
4211 | if (HAS_BSD(dev)) { |
4248 | ret = intel_init_bsd_ring_buffer(dev); |
4212 | ret = intel_init_bsd_ring_buffer(dev); |
4249 | if (ret) |
4213 | if (ret) |
Line 4250... | Line 4214... | ||
4250 | goto cleanup_render_ring; |
4214 | goto cleanup_render_ring; |
4251 | } |
4215 | } |
4252 | 4216 | ||
4253 | if (intel_enable_blt(dev)) { |
4217 | if (HAS_BLT(dev)) { |
4254 | ret = intel_init_blt_ring_buffer(dev); |
4218 | ret = intel_init_blt_ring_buffer(dev); |
Line 4266... | Line 4230... | ||
4266 | ret = intel_init_bsd2_ring_buffer(dev); |
4230 | ret = intel_init_bsd2_ring_buffer(dev); |
4267 | if (ret) |
4231 | if (ret) |
4268 | goto cleanup_vebox_ring; |
4232 | goto cleanup_vebox_ring; |
4269 | } |
4233 | } |
Line 4270... | Line -... | ||
4270 | - | ||
4271 | ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); |
- | |
4272 | if (ret) |
- | |
4273 | goto cleanup_bsd2_ring; |
- | |
4274 | 4234 | ||
Line 4275... | Line -... | ||
4275 | return 0; |
- | |
4276 | - | ||
4277 | cleanup_bsd2_ring: |
4235 | return 0; |
4278 | intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]); |
4236 | |
4279 | cleanup_vebox_ring: |
4237 | cleanup_vebox_ring: |
4280 | intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); |
4238 | intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); |
4281 | cleanup_blt_ring: |
4239 | cleanup_blt_ring: |
Line 4290... | Line 4248... | ||
4290 | 4248 | ||
4291 | int |
4249 | int |
4292 | i915_gem_init_hw(struct drm_device *dev) |
4250 | i915_gem_init_hw(struct drm_device *dev) |
4293 | { |
4251 | { |
- | 4252 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
4294 | struct drm_i915_private *dev_priv = dev->dev_private; |
4253 | struct intel_engine_cs *ring; |
Line 4295... | Line 4254... | ||
4295 | int ret, i; |
4254 | int ret, i, j; |
4296 | 4255 | ||
Line -... | Line 4256... | ||
- | 4256 | if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) |
|
- | 4257 | return -EIO; |
|
- | 4258 | ||
4297 | if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) |
4259 | /* Double layer security blanket, see i915_gem_init() */ |
4298 | return -EIO; |
4260 | intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); |
Line 4299... | Line 4261... | ||
4299 | 4261 | ||
4300 | if (dev_priv->ellc_size) |
4262 | if (dev_priv->ellc_size) |
Line 4316... | Line 4278... | ||
4316 | } |
4278 | } |
4317 | } |
4279 | } |
Line 4318... | Line 4280... | ||
4318 | 4280 | ||
Line -... | Line 4281... | ||
- | 4281 | i915_gem_init_swizzling(dev); |
|
- | 4282 | ||
- | 4283 | /* |
|
- | 4284 | * At least 830 can leave some of the unused rings |
|
- | 4285 | * "active" (ie. head != tail) after resume which |
|
- | 4286 | * will prevent c3 entry. Makes sure all unused rings |
|
- | 4287 | * are totally idle. |
|
- | 4288 | */ |
|
- | 4289 | init_unused_rings(dev); |
|
- | 4290 | ||
4319 | i915_gem_init_swizzling(dev); |
4291 | BUG_ON(!dev_priv->ring[RCS].default_context); |
- | 4292 | ||
- | 4293 | ret = i915_ppgtt_init_hw(dev); |
|
- | 4294 | if (ret) { |
|
- | 4295 | DRM_ERROR("PPGTT enable HW failed %d\n", ret); |
|
- | 4296 | goto out; |
|
- | 4297 | } |
|
- | 4298 | ||
- | 4299 | /* Need to do basic initialisation of all rings first: */ |
|
4320 | 4300 | for_each_ring(ring, dev_priv, i) { |
|
4321 | ret = dev_priv->gt.init_rings(dev); |
4301 | ret = ring->init_hw(ring); |
- | 4302 | if (ret) |
|
Line -... | Line 4303... | ||
- | 4303 | goto out; |
|
- | 4304 | } |
|
4322 | if (ret) |
4305 | |
- | 4306 | /* We can't enable contexts until all firmware is loaded */ |
|
- | 4307 | if (HAS_GUC_UCODE(dev)) { |
|
- | 4308 | ret = intel_guc_ucode_load(dev); |
|
- | 4309 | if (ret) { |
|
- | 4310 | /* |
|
- | 4311 | * If we got an error and GuC submission is enabled, map |
|
- | 4312 | * the error to -EIO so the GPU will be declared wedged. |
|
- | 4313 | * OTOH, if we didn't intend to use the GuC anyway, just |
|
- | 4314 | * discard the error and carry on. |
|
- | 4315 | */ |
|
4323 | return ret; |
4316 | DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret, |
- | 4317 | i915.enable_guc_submission ? "" : |
|
- | 4318 | " (ignored)"); |
|
- | 4319 | ret = i915.enable_guc_submission ? -EIO : 0; |
|
- | 4320 | if (ret) |
|
Line 4324... | Line 4321... | ||
4324 | 4321 | goto out; |
|
4325 | for (i = 0; i < NUM_L3_SLICES(dev); i++) |
- | |
4326 | i915_gem_l3_remap(&dev_priv->ring[RCS], i); |
4322 | } |
4327 | - | ||
4328 | /* |
4323 | } |
4329 | * XXX: Contexts should only be initialized once. Doing a switch to the |
- | |
4330 | * default context switch however is something we'd like to do after |
4324 | |
4331 | * reset or thaw (the latter may not actually be necessary for HW, but |
4325 | /* |
- | 4326 | * Increment the next seqno by 0x100 so we have a visible break |
|
- | 4327 | * on re-initialisation |
|
- | 4328 | */ |
|
- | 4329 | ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100); |
|
4332 | * goes with our code better). Context switching requires rings (for |
4330 | if (ret) |
- | 4331 | goto out; |
|
- | 4332 | ||
4333 | * the do_switch), but before enabling PPGTT. So don't move this. |
4333 | /* Now it is safe to go back round and do everything else: */ |
- | 4334 | for_each_ring(ring, dev_priv, i) { |
|
- | 4335 | struct drm_i915_gem_request *req; |
|
- | 4336 | ||
4334 | */ |
4337 | WARN_ON(!ring->default_context); |
- | 4338 | ||
- | 4339 | ret = i915_gem_request_alloc(ring, ring->default_context, &req); |
|
Line 4335... | Line 4340... | ||
4335 | ret = i915_gem_context_enable(dev_priv); |
4340 | if (ret) { |
- | 4341 | i915_gem_cleanup_ringbuffer(dev); |
|
- | 4342 | goto out; |
|
4336 | if (ret && ret != -EIO) { |
4343 | } |
Line 4337... | Line 4344... | ||
4337 | DRM_ERROR("Context enable failed %d\n", ret); |
4344 | |
- | 4345 | if (ring->id == RCS) { |
|
- | 4346 | for (j = 0; j < NUM_L3_SLICES(dev); j++) |
|
- | 4347 | i915_gem_l3_remap(req, j); |
|
- | 4348 | } |
|
- | 4349 | ||
- | 4350 | ret = i915_ppgtt_init_ring(req); |
|
- | 4351 | if (ret && ret != -EIO) { |
|
- | 4352 | DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret); |
|
4338 | i915_gem_cleanup_ringbuffer(dev); |
4353 | i915_gem_request_cancel(req); |
4339 | 4354 | i915_gem_cleanup_ringbuffer(dev); |
|
- | 4355 | goto out; |
|
4340 | return ret; |
4356 | } |
- | 4357 | ||
4341 | } |
4358 | ret = i915_gem_context_enable(req); |
Line -... | Line 4359... | ||
- | 4359 | if (ret && ret != -EIO) { |
|
- | 4360 | DRM_ERROR("Context enable ring #%d failed %d\n", i, ret); |
|
- | 4361 | i915_gem_request_cancel(req); |
|
- | 4362 | i915_gem_cleanup_ringbuffer(dev); |
|
- | 4363 | goto out; |
|
4342 | 4364 | } |
|
4343 | ret = i915_ppgtt_init_hw(dev); |
4365 | |
Line 4344... | Line 4366... | ||
4344 | if (ret && ret != -EIO) { |
4366 | i915_add_request_no_flush(req); |
4345 | DRM_ERROR("PPGTT enable failed %d\n", ret); |
4367 | } |
Line 4366... | Line 4388... | ||
4366 | VLV_GTLC_ALLOWWAKEACK), 10)) |
4388 | VLV_GTLC_ALLOWWAKEACK), 10)) |
4367 | DRM_DEBUG_DRIVER("allow wake ack timed out\n"); |
4389 | DRM_DEBUG_DRIVER("allow wake ack timed out\n"); |
4368 | } |
4390 | } |
Line 4369... | Line 4391... | ||
4369 | 4391 | ||
4370 | if (!i915.enable_execlists) { |
4392 | if (!i915.enable_execlists) { |
4371 | dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission; |
4393 | dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission; |
4372 | dev_priv->gt.init_rings = i915_gem_init_rings; |
4394 | dev_priv->gt.init_rings = i915_gem_init_rings; |
4373 | dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer; |
4395 | dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer; |
4374 | dev_priv->gt.stop_ring = intel_stop_ring_buffer; |
4396 | dev_priv->gt.stop_ring = intel_stop_ring_buffer; |
4375 | } else { |
4397 | } else { |
4376 | dev_priv->gt.do_execbuf = intel_execlists_submission; |
4398 | dev_priv->gt.execbuf_submit = intel_execlists_submission; |
4377 | dev_priv->gt.init_rings = intel_logical_rings_init; |
4399 | dev_priv->gt.init_rings = intel_logical_rings_init; |
4378 | dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup; |
4400 | dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup; |
4379 | dev_priv->gt.stop_ring = intel_logical_ring_stop; |
4401 | dev_priv->gt.stop_ring = intel_logical_ring_stop; |
Line -... | Line 4402... | ||
- | 4402 | } |
|
- | 4403 | ||
- | 4404 | /* This is just a security blanket to placate dragons. |
|
- | 4405 | * On some systems, we very sporadically observe that the first TLBs |
|
- | 4406 | * used by the CS may be stale, despite us poking the TLB reset. If |
|
- | 4407 | * we hold the forcewake during initialisation these problems |
|
- | 4408 | * just magically go away. |
|
- | 4409 | */ |
|
4380 | } |
4410 | intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); |
4381 | 4411 | ||
4382 | // ret = i915_gem_init_userptr(dev); |
- | |
4383 | // if (ret) { |
4412 | // ret = i915_gem_init_userptr(dev); |
4384 | // mutex_unlock(&dev->struct_mutex); |
- | |
Line 4385... | Line 4413... | ||
4385 | // return ret; |
4413 | // if (ret) |
Line 4386... | Line 4414... | ||
4386 | // } |
4414 | // goto out_unlock; |
4387 | 4415 | ||
- | 4416 | i915_gem_init_global_gtt(dev); |
|
- | 4417 | ||
4388 | i915_gem_init_global_gtt(dev); |
4418 | ret = i915_gem_context_init(dev); |
4389 | 4419 | if (ret) |
|
4390 | ret = i915_gem_context_init(dev); |
4420 | goto out_unlock; |
Line 4391... | Line 4421... | ||
4391 | if (ret) { |
4421 | |
4392 | mutex_unlock(&dev->struct_mutex); |
4422 | ret = dev_priv->gt.init_rings(dev); |
4393 | return ret; |
4423 | if (ret) |
4394 | } |
4424 | goto out_unlock; |
4395 | 4425 | ||
4396 | ret = i915_gem_init_hw(dev); |
4426 | ret = i915_gem_init_hw(dev); |
4397 | if (ret == -EIO) { |
4427 | if (ret == -EIO) { |
4398 | /* Allow ring initialisation to fail by marking the GPU as |
4428 | /* Allow ring initialisation to fail by marking the GPU as |
4399 | * wedged. But we only want to do this where the GPU is angry, |
4429 | * wedged. But we only want to do this where the GPU is angry, |
4400 | * for all other failure, such as an allocation failure, bail. |
4430 | * for all other failure, such as an allocation failure, bail. |
- | 4431 | */ |
|
- | 4432 | DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); |
|
- | 4433 | atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); |
|
4401 | */ |
4434 | ret = 0; |
Line 4402... | Line 4435... | ||
4402 | DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); |
4435 | } |
4403 | atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); |
4436 | |
Line 4424... | Line 4457... | ||
4424 | { |
4457 | { |
4425 | INIT_LIST_HEAD(&ring->active_list); |
4458 | INIT_LIST_HEAD(&ring->active_list); |
4426 | INIT_LIST_HEAD(&ring->request_list); |
4459 | INIT_LIST_HEAD(&ring->request_list); |
4427 | } |
4460 | } |
Line 4428... | Line -... | ||
4428 | - | ||
4429 | void i915_init_vm(struct drm_i915_private *dev_priv, |
- | |
4430 | struct i915_address_space *vm) |
- | |
4431 | { |
- | |
4432 | if (!i915_is_ggtt(vm)) |
- | |
4433 | drm_mm_init(&vm->mm, vm->start, vm->total); |
- | |
4434 | vm->dev = dev_priv->dev; |
- | |
4435 | INIT_LIST_HEAD(&vm->active_list); |
- | |
4436 | INIT_LIST_HEAD(&vm->inactive_list); |
- | |
4437 | INIT_LIST_HEAD(&vm->global_link); |
- | |
4438 | list_add_tail(&vm->global_link, &dev_priv->vm_list); |
- | |
4439 | } |
- | |
4440 | 4461 | ||
4441 | void |
4462 | void |
4442 | i915_gem_load(struct drm_device *dev) |
4463 | i915_gem_load(struct drm_device *dev) |
4443 | { |
4464 | { |
4444 | struct drm_i915_private *dev_priv = dev->dev_private; |
4465 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 4445... | Line 4466... | ||
4445 | int i; |
4466 | int i; |
4446 | - | ||
4447 | INIT_LIST_HEAD(&dev_priv->vm_list); |
- | |
4448 | i915_init_vm(dev_priv, &dev_priv->gtt.base); |
4467 | |
4449 | 4468 | INIT_LIST_HEAD(&dev_priv->vm_list); |
|
4450 | INIT_LIST_HEAD(&dev_priv->context_list); |
4469 | INIT_LIST_HEAD(&dev_priv->context_list); |
4451 | INIT_LIST_HEAD(&dev_priv->mm.unbound_list); |
4470 | INIT_LIST_HEAD(&dev_priv->mm.unbound_list); |
4452 | INIT_LIST_HEAD(&dev_priv->mm.bound_list); |
4471 | INIT_LIST_HEAD(&dev_priv->mm.bound_list); |
Line 4459... | Line 4478... | ||
4459 | i915_gem_retire_work_handler); |
4478 | i915_gem_retire_work_handler); |
4460 | INIT_DELAYED_WORK(&dev_priv->mm.idle_work, |
4479 | INIT_DELAYED_WORK(&dev_priv->mm.idle_work, |
4461 | i915_gem_idle_work_handler); |
4480 | i915_gem_idle_work_handler); |
4462 | init_waitqueue_head(&dev_priv->gpu_error.reset_queue); |
4481 | init_waitqueue_head(&dev_priv->gpu_error.reset_queue); |
Line 4463... | Line -... | ||
4463 | - | ||
4464 | /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ |
- | |
4465 | if (!drm_core_check_feature(dev, DRIVER_MODESET) && IS_GEN3(dev)) { |
- | |
4466 | I915_WRITE(MI_ARB_STATE, |
- | |
4467 | _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); |
- | |
4468 | } |
- | |
4469 | 4482 | ||
Line 4470... | Line -... | ||
4470 | dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
- | |
4471 | - | ||
4472 | /* Old X drivers will take 0-2 for front, back, depth buffers */ |
- | |
4473 | if (!drm_core_check_feature(dev, DRIVER_MODESET)) |
- | |
4474 | dev_priv->fence_reg_start = 3; |
4483 | dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
4475 | 4484 | ||
4476 | if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) |
4485 | if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) |
4477 | dev_priv->num_fence_regs = 32; |
4486 | dev_priv->num_fence_regs = 32; |
4478 | else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
4487 | else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
4479 | dev_priv->num_fence_regs = 16; |
4488 | dev_priv->num_fence_regs = 16; |
Line -... | Line 4489... | ||
- | 4489 | else |
|
- | 4490 | dev_priv->num_fence_regs = 8; |
|
- | 4491 | ||
- | 4492 | if (intel_vgpu_active(dev)) |
|
- | 4493 | dev_priv->num_fence_regs = |
|
- | 4494 | I915_READ(vgtif_reg(avail_rs.fence_num)); |
|
- | 4495 | ||
- | 4496 | /* |
|
- | 4497 | * Set initial sequence number for requests. |
|
- | 4498 | * Using this number allows the wraparound to happen early, |
|
- | 4499 | * catching any obvious problems. |
|
- | 4500 | */ |
|
4480 | else |
4501 | dev_priv->next_seqno = ((u32)~0 - 0x1100); |
4481 | dev_priv->num_fence_regs = 8; |
4502 | dev_priv->last_seqno = ((u32)~0 - 0x1101); |
4482 | 4503 | ||
Line 4483... | Line 4504... | ||
4483 | /* Initialize fence registers to zero */ |
4504 | /* Initialize fence registers to zero */ |
Line 4489... | Line 4510... | ||
4489 | dev_priv->mm.interruptible = true; |
4510 | dev_priv->mm.interruptible = true; |
Line 4490... | Line 4511... | ||
4490 | 4511 | ||
4491 | mutex_init(&dev_priv->fb_tracking.lock); |
4512 | mutex_init(&dev_priv->fb_tracking.lock); |
Line -... | Line 4513... | ||
- | 4513 | } |
|
- | 4514 | ||
- | 4515 | void i915_gem_release(struct drm_device *dev, struct drm_file *file) |
|
- | 4516 | { |
|
- | 4517 | struct drm_i915_file_private *file_priv = file->driver_priv; |
|
- | 4518 | ||
- | 4519 | /* Clean up our request list when the client is going away, so that |
|
- | 4520 | * later retire_requests won't dereference our soon-to-be-gone |
|
- | 4521 | * file_priv. |
|
- | 4522 | */ |
|
- | 4523 | spin_lock(&file_priv->mm.lock); |
|
- | 4524 | while (!list_empty(&file_priv->mm.request_list)) { |
|
- | 4525 | struct drm_i915_gem_request *request; |
|
- | 4526 | ||
- | 4527 | request = list_first_entry(&file_priv->mm.request_list, |
|
- | 4528 | struct drm_i915_gem_request, |
|
- | 4529 | client_list); |
|
- | 4530 | list_del(&request->client_list); |
|
- | 4531 | request->file_priv = NULL; |
|
- | 4532 | } |
|
- | 4533 | spin_unlock(&file_priv->mm.lock); |
|
- | 4534 | ||
- | 4535 | if (!list_empty(&file_priv->rps.link)) { |
|
- | 4536 | spin_lock(&to_i915(dev)->rps.client_lock); |
|
- | 4537 | list_del(&file_priv->rps.link); |
|
- | 4538 | spin_unlock(&to_i915(dev)->rps.client_lock); |
|
- | 4539 | } |
|
4492 | } |
4540 | } |
4493 | 4541 | ||
4494 | int i915_gem_open(struct drm_device *dev, struct drm_file *file) |
4542 | int i915_gem_open(struct drm_device *dev, struct drm_file *file) |
4495 | { |
4543 | { |
Line 4503... | Line 4551... | ||
4503 | return -ENOMEM; |
4551 | return -ENOMEM; |
Line 4504... | Line 4552... | ||
4504 | 4552 | ||
4505 | file->driver_priv = file_priv; |
4553 | file->driver_priv = file_priv; |
4506 | file_priv->dev_priv = dev->dev_private; |
4554 | file_priv->dev_priv = dev->dev_private; |
- | 4555 | file_priv->file = file; |
|
Line 4507... | Line 4556... | ||
4507 | file_priv->file = file; |
4556 | INIT_LIST_HEAD(&file_priv->rps.link); |
4508 | 4557 | ||
4509 | spin_lock_init(&file_priv->mm.lock); |
- | |
4510 | INIT_LIST_HEAD(&file_priv->mm.request_list); |
- | |
Line 4511... | Line 4558... | ||
4511 | // INIT_DELAYED_WORK(&file_priv->mm.idle_work, |
4558 | spin_lock_init(&file_priv->mm.lock); |
4512 | // i915_gem_file_idle_work_handler); |
4559 | INIT_LIST_HEAD(&file_priv->mm.request_list); |
4513 | 4560 | ||
Line 4514... | Line 4561... | ||
4514 | ret = i915_gem_context_open(dev, file); |
4561 | ret = i915_gem_context_open(dev, file); |
4515 | if (ret) |
4562 | if (ret) |
Line 4516... | Line 4563... | ||
4516 | kfree(file_priv); |
4563 | kfree(file_priv); |
4517 | 4564 | ||
4518 | return ret; |
4565 | return ret; |
4519 | } |
4566 | } |
4520 | 4567 | ||
4521 | /** |
4568 | /** |
4522 | * i915_gem_track_fb - update frontbuffer tracking |
4569 | * i915_gem_track_fb - update frontbuffer tracking |
4523 | * old: current GEM buffer for the frontbuffer slots |
4570 | * @old: current GEM buffer for the frontbuffer slots |
4524 | * new: new GEM buffer for the frontbuffer slots |
4571 | * @new: new GEM buffer for the frontbuffer slots |
4525 | * frontbuffer_bits: bitmask of frontbuffer slots |
4572 | * @frontbuffer_bits: bitmask of frontbuffer slots |
Line 4542... | Line 4589... | ||
4542 | WARN_ON(new->frontbuffer_bits & frontbuffer_bits); |
4589 | WARN_ON(new->frontbuffer_bits & frontbuffer_bits); |
4543 | new->frontbuffer_bits |= frontbuffer_bits; |
4590 | new->frontbuffer_bits |= frontbuffer_bits; |
4544 | } |
4591 | } |
4545 | } |
4592 | } |
Line 4546... | Line -... | ||
4546 | - | ||
4547 | static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) |
- | |
4548 | { |
- | |
4549 | if (!mutex_is_locked(mutex)) |
- | |
4550 | return false; |
- | |
4551 | - | ||
4552 | #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) |
- | |
4553 | return mutex->owner == task; |
- | |
4554 | #else |
- | |
4555 | /* Since UP may be pre-empted, we cannot assume that we own the lock */ |
- | |
4556 | return false; |
- | |
4557 | #endif |
- | |
4558 | } |
- | |
4559 | 4593 | ||
4560 | /* All the new VM stuff */ |
4594 | /* All the new VM stuff */ |
4561 | unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o, |
4595 | u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, |
4562 | struct i915_address_space *vm) |
4596 | struct i915_address_space *vm) |
4563 | { |
4597 | { |
4564 | struct drm_i915_private *dev_priv = o->base.dev->dev_private; |
4598 | struct drm_i915_private *dev_priv = o->base.dev->dev_private; |
Line 4565... | Line 4599... | ||
4565 | struct i915_vma *vma; |
4599 | struct i915_vma *vma; |
Line 4566... | Line 4600... | ||
4566 | 4600 | ||
- | 4601 | WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); |
|
- | 4602 | ||
- | 4603 | list_for_each_entry(vma, &o->vma_list, vma_link) { |
|
4567 | WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); |
4604 | if (i915_is_ggtt(vma->vm) && |
4568 | 4605 | vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) |
|
4569 | list_for_each_entry(vma, &o->vma_list, vma_link) { |
- | |
4570 | if (vma->vm == vm) |
4606 | continue; |
- | 4607 | if (vma->vm == vm) |
|
4571 | return vma->node.start; |
4608 | return vma->node.start; |
4572 | 4609 | } |
|
4573 | } |
4610 | |
4574 | WARN(1, "%s vma for this object not found.\n", |
4611 | WARN(1, "%s vma for this object not found.\n", |
Line -... | Line 4612... | ||
- | 4612 | i915_is_ggtt(vm) ? "global" : "ppgtt"); |
|
- | 4613 | return -1; |
|
- | 4614 | } |
|
- | 4615 | ||
- | 4616 | u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, |
|
- | 4617 | const struct i915_ggtt_view *view) |
|
- | 4618 | { |
|
- | 4619 | struct i915_address_space *ggtt = i915_obj_to_ggtt(o); |
|
- | 4620 | struct i915_vma *vma; |
|
- | 4621 | ||
- | 4622 | list_for_each_entry(vma, &o->vma_list, vma_link) |
|
- | 4623 | if (vma->vm == ggtt && |
|
- | 4624 | i915_ggtt_view_equal(&vma->ggtt_view, view)) |
|
- | 4625 | return vma->node.start; |
|
- | 4626 | ||
4575 | i915_is_ggtt(vm) ? "global" : "ppgtt"); |
4627 | WARN(1, "global vma for this object not found. (view=%u)\n", view->type); |
4576 | return -1; |
4628 | return -1; |
4577 | } |
4629 | } |
4578 | 4630 | ||
Line 4579... | Line 4631... | ||
4579 | bool i915_gem_obj_bound(struct drm_i915_gem_object *o, |
4631 | bool i915_gem_obj_bound(struct drm_i915_gem_object *o, |
- | 4632 | struct i915_address_space *vm) |
|
- | 4633 | { |
|
- | 4634 | struct i915_vma *vma; |
|
4580 | struct i915_address_space *vm) |
4635 | |
4581 | { |
4636 | list_for_each_entry(vma, &o->vma_list, vma_link) { |
- | 4637 | if (i915_is_ggtt(vma->vm) && |
|
- | 4638 | vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) |
|
- | 4639 | continue; |
|
- | 4640 | if (vma->vm == vm && drm_mm_node_allocated(&vma->node)) |
|
- | 4641 | return true; |
|
- | 4642 | } |
|
- | 4643 | ||
- | 4644 | return false; |
|
- | 4645 | } |
|
- | 4646 | ||
- | 4647 | bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o, |
|
- | 4648 | const struct i915_ggtt_view *view) |
|
- | 4649 | { |
|
- | 4650 | struct i915_address_space *ggtt = i915_obj_to_ggtt(o); |
|
- | 4651 | struct i915_vma *vma; |
|
- | 4652 | ||
Line 4582... | Line 4653... | ||
4582 | struct i915_vma *vma; |
4653 | list_for_each_entry(vma, &o->vma_list, vma_link) |
4583 | 4654 | if (vma->vm == ggtt && |
|
Line 4584... | Line 4655... | ||
4584 | list_for_each_entry(vma, &o->vma_list, vma_link) |
4655 | i915_ggtt_view_equal(&vma->ggtt_view, view) && |
Line 4607... | Line 4678... | ||
4607 | 4678 | ||
Line 4608... | Line 4679... | ||
4608 | WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); |
4679 | WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); |
Line 4609... | Line 4680... | ||
4609 | 4680 | ||
- | 4681 | BUG_ON(list_empty(&o->vma_list)); |
|
- | 4682 | ||
- | 4683 | list_for_each_entry(vma, &o->vma_list, vma_link) { |
|
4610 | BUG_ON(list_empty(&o->vma_list)); |
4684 | if (i915_is_ggtt(vma->vm) && |
4611 | 4685 | vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) |
|
4612 | list_for_each_entry(vma, &o->vma_list, vma_link) |
4686 | continue; |
4613 | if (vma->vm == vm) |
4687 | if (vma->vm == vm) |
4614 | return vma->node.size; |
4688 | return vma->node.size; |
Line -... | Line 4689... | ||
- | 4689 | } |
|
- | 4690 | return 0; |
|
- | 4691 | } |
|
- | 4692 | ||
- | 4693 | bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) |
|
- | 4694 | { |
|
Line -... | Line 4695... | ||
- | 4695 | struct i915_vma *vma; |
|
- | 4696 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
|
Line -... | Line 4697... | ||
- | 4697 | if (vma->pin_count > 0) |
|
- | 4698 | return true; |
|
4615 | 4699 | ||
- | 4700 | return false; |
|
4616 | return 0; |
4701 | } |
- | 4702 | ||
4617 | } |
4703 | /* Allocate a new GEM object and fill it with the supplied data */ |
- | 4704 | struct drm_i915_gem_object * |
|
- | 4705 | i915_gem_object_create_from_data(struct drm_device *dev, |
|
Line 4618... | Line 4706... | ||
4618 | 4706 | const void *data, size_t size) |
|
4619 | 4707 | { |
|
4620 | 4708 | struct drm_i915_gem_object *obj; |
|
Line -... | Line 4709... | ||
- | 4709 | struct sg_table *sg; |
|
- | 4710 | size_t bytes; |
|
- | 4711 | int ret; |
|
- | 4712 | ||
- | 4713 | obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE)); |
|
- | 4714 | if (IS_ERR_OR_NULL(obj)) |
|
- | 4715 | return obj; |
|
- | 4716 | ||
- | 4717 | ret = i915_gem_object_set_to_cpu_domain(obj, true); |
|
- | 4718 | if (ret) |
|
- | 4719 | goto fail; |
|
- | 4720 | ||
- | 4721 | ret = i915_gem_object_get_pages(obj); |
|
- | 4722 | if (ret) |
|
- | 4723 | goto fail; |
|
- | 4724 | ||
- | 4725 | i915_gem_object_pin_pages(obj); |
|
- | 4726 | sg = obj->pages; |
|
- | 4727 | bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); |
|
4621 | struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj) |
4728 | i915_gem_object_unpin_pages(obj); |
- | 4729 | ||
- | 4730 | if (WARN_ON(bytes != size)) { |
|
- | 4731 | DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size); |
|
- | 4732 | ret = -EFAULT; |
|
4622 | { |
4733 | goto fail; |