Rev 3746 | Rev 4246 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 3746 | Rev 4104 | ||
---|---|---|---|
Line 69... | Line 69... | ||
69 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
69 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
70 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
70 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
71 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
71 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
Line 72... | Line 72... | ||
72 | 72 | ||
73 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
73 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
- | 74 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, |
|
- | 75 | bool force); |
|
74 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
76 | static __must_check int |
- | 77 | i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, |
|
75 | static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
78 | struct i915_address_space *vm, |
76 | unsigned alignment, |
79 | unsigned alignment, |
77 | bool map_and_fenceable, |
80 | bool map_and_fenceable, |
78 | bool nonblocking); |
81 | bool nonblocking); |
79 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
82 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
Line 86... | Line 89... | ||
86 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
89 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
87 | struct drm_i915_fence_reg *fence, |
90 | struct drm_i915_fence_reg *fence, |
88 | bool enable); |
91 | bool enable); |
Line 89... | Line 92... | ||
89 | 92 | ||
90 | static long i915_gem_purge(struct drm_i915_private *dev_priv, long target); |
93 | static long i915_gem_purge(struct drm_i915_private *dev_priv, long target); |
91 | static void i915_gem_shrink_all(struct drm_i915_private *dev_priv); |
94 | static long i915_gem_shrink_all(struct drm_i915_private *dev_priv); |
Line -... | Line 95... | ||
- | 95 | static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); |
|
- | 96 | ||
- | 97 | static bool cpu_cache_is_coherent(struct drm_device *dev, |
|
- | 98 | enum i915_cache_level level) |
|
- | 99 | { |
|
- | 100 | return HAS_LLC(dev) || level != I915_CACHE_NONE; |
|
- | 101 | } |
|
- | 102 | ||
- | 103 | static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) |
|
- | 104 | { |
|
- | 105 | if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) |
|
- | 106 | return true; |
|
- | 107 | ||
- | 108 | return obj->pin_display; |
|
92 | static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); |
109 | } |
93 | 110 | ||
94 | static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) |
111 | static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) |
95 | { |
112 | { |
Line 105... | Line 122... | ||
105 | 122 | ||
106 | /* some bookkeeping */ |
123 | /* some bookkeeping */ |
107 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
124 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
108 | size_t size) |
125 | size_t size) |
- | 126 | { |
|
109 | { |
127 | spin_lock(&dev_priv->mm.object_stat_lock); |
110 | dev_priv->mm.object_count++; |
128 | dev_priv->mm.object_count++; |
- | 129 | dev_priv->mm.object_memory += size; |
|
111 | dev_priv->mm.object_memory += size; |
130 | spin_unlock(&dev_priv->mm.object_stat_lock); |
Line 112... | Line 131... | ||
112 | } |
131 | } |
113 | 132 | ||
114 | static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, |
133 | static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, |
- | 134 | size_t size) |
|
115 | size_t size) |
135 | { |
116 | { |
136 | spin_lock(&dev_priv->mm.object_stat_lock); |
- | 137 | dev_priv->mm.object_count--; |
|
117 | dev_priv->mm.object_count--; |
138 | dev_priv->mm.object_memory -= size; |
Line 118... | Line 139... | ||
118 | dev_priv->mm.object_memory -= size; |
139 | spin_unlock(&dev_priv->mm.object_stat_lock); |
119 | } |
140 | } |
120 | 141 | ||
Line 166... | Line 187... | ||
166 | } |
187 | } |
Line 167... | Line 188... | ||
167 | 188 | ||
168 | static inline bool |
189 | static inline bool |
169 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
190 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
170 | { |
191 | { |
171 | return obj->gtt_space && !obj->active; |
192 | return i915_gem_obj_bound_any(obj) && !obj->active; |
Line 172... | Line 193... | ||
172 | } |
193 | } |
Line 211... | Line 232... | ||
211 | struct drm_i915_gem_object *obj; |
232 | struct drm_i915_gem_object *obj; |
212 | size_t pinned; |
233 | size_t pinned; |
Line 213... | Line 234... | ||
213 | 234 | ||
214 | pinned = 0; |
235 | pinned = 0; |
215 | mutex_lock(&dev->struct_mutex); |
236 | mutex_lock(&dev->struct_mutex); |
216 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) |
237 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) |
217 | if (obj->pin_count) |
238 | if (obj->pin_count) |
218 | pinned += obj->gtt_space->size; |
239 | pinned += i915_gem_obj_ggtt_size(obj); |
Line 219... | Line 240... | ||
219 | mutex_unlock(&dev->struct_mutex); |
240 | mutex_unlock(&dev->struct_mutex); |
220 | 241 | ||
Line 221... | Line 242... | ||
221 | args->aper_size = dev_priv->gtt.total; |
242 | args->aper_size = dev_priv->gtt.base.total; |
222 | args->aper_available_size = args->aper_size - pinned; |
243 | args->aper_available_size = args->aper_size - pinned; |
Line 254... | Line 275... | ||
254 | obj = i915_gem_alloc_object(dev, size); |
275 | obj = i915_gem_alloc_object(dev, size); |
255 | if (obj == NULL) |
276 | if (obj == NULL) |
256 | return -ENOMEM; |
277 | return -ENOMEM; |
Line 257... | Line 278... | ||
257 | 278 | ||
258 | ret = drm_gem_handle_create(file, &obj->base, &handle); |
- | |
259 | if (ret) { |
- | |
260 | drm_gem_object_release(&obj->base); |
- | |
261 | i915_gem_info_remove_obj(dev->dev_private, obj->base.size); |
- | |
262 | kfree(obj); |
- | |
263 | return ret; |
- | |
264 | } |
- | |
265 | 279 | ret = drm_gem_handle_create(file, &obj->base, &handle); |
|
266 | /* drop reference from allocate - handle holds it now */ |
280 | /* drop reference from allocate - handle holds it now */ |
- | 281 | drm_gem_object_unreference_unlocked(&obj->base); |
|
267 | drm_gem_object_unreference(&obj->base); |
282 | if (ret) |
Line 268... | Line 283... | ||
268 | trace_i915_gem_object_create(obj); |
283 | return ret; |
269 | 284 | ||
270 | *handle_p = handle; |
285 | *handle_p = handle; |
Line 281... | Line 296... | ||
281 | args->size = args->pitch * args->height; |
296 | args->size = args->pitch * args->height; |
282 | return i915_gem_create(file, dev, |
297 | return i915_gem_create(file, dev, |
283 | args->size, &args->handle); |
298 | args->size, &args->handle); |
284 | } |
299 | } |
Line 285... | Line -... | ||
285 | - | ||
286 | int i915_gem_dumb_destroy(struct drm_file *file, |
- | |
287 | struct drm_device *dev, |
- | |
288 | uint32_t handle) |
- | |
289 | { |
- | |
290 | return drm_gem_handle_delete(file, handle); |
- | |
291 | } |
- | |
292 | 300 | ||
293 | /** |
301 | /** |
294 | * Creates a new mm object and returns a handle to it. |
302 | * Creates a new mm object and returns a handle to it. |
295 | */ |
303 | */ |
296 | int |
304 | int |
Line 458... | Line 466... | ||
458 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { |
466 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { |
459 | /* If we're not in the cpu read domain, set ourself into the gtt |
467 | /* If we're not in the cpu read domain, set ourself into the gtt |
460 | * read domain and manually flush cachelines (if required). This |
468 | * read domain and manually flush cachelines (if required). This |
461 | * optimizes for the case when the gpu will dirty the data |
469 | * optimizes for the case when the gpu will dirty the data |
462 | * anyway again before the next pread happens. */ |
470 | * anyway again before the next pread happens. */ |
463 | if (obj->cache_level == I915_CACHE_NONE) |
471 | needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level); |
464 | needs_clflush = 1; |
- | |
465 | if (obj->gtt_space) { |
472 | if (i915_gem_obj_bound_any(obj)) { |
466 | ret = i915_gem_object_set_to_gtt_domain(obj, false); |
473 | ret = i915_gem_object_set_to_gtt_domain(obj, false); |
467 | if (ret) |
474 | if (ret) |
468 | return ret; |
475 | return ret; |
469 | } |
476 | } |
470 | } |
477 | } |
Line 503... | Line 510... | ||
503 | if (ret == 0) |
510 | if (ret == 0) |
504 | goto next_page; |
511 | goto next_page; |
Line 505... | Line 512... | ||
505 | 512 | ||
Line 506... | Line 513... | ||
506 | mutex_unlock(&dev->struct_mutex); |
513 | mutex_unlock(&dev->struct_mutex); |
507 | 514 | ||
508 | if (!prefaulted) { |
515 | if (likely(!i915_prefault_disable) && !prefaulted) { |
509 | ret = fault_in_multipages_writeable(user_data, remain); |
516 | ret = fault_in_multipages_writeable(user_data, remain); |
510 | /* Userspace is tricking us, but we've already clobbered |
517 | /* Userspace is tricking us, but we've already clobbered |
511 | * its pages with the prefault and promised to write the |
518 | * its pages with the prefault and promised to write the |
Line 635... | Line 642... | ||
635 | loff_t offset, page_base; |
642 | loff_t offset, page_base; |
636 | char __user *user_data; |
643 | char __user *user_data; |
637 | int page_offset, page_length, ret; |
644 | int page_offset, page_length, ret; |
638 | char *vaddr; |
645 | char *vaddr; |
Line 639... | Line 646... | ||
639 | 646 | ||
640 | ret = i915_gem_object_pin(obj, 0, true, true); |
647 | ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); |
641 | if (ret) |
648 | if (ret) |
Line 642... | Line 649... | ||
642 | goto out; |
649 | goto out; |
643 | 650 | ||
Line 657... | Line 664... | ||
657 | }; |
664 | }; |
Line 658... | Line 665... | ||
658 | 665 | ||
659 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
666 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
Line 660... | Line 667... | ||
660 | remain = args->size; |
667 | remain = args->size; |
Line 661... | Line 668... | ||
661 | 668 | ||
662 | offset = obj->gtt_offset + args->offset; |
669 | offset = i915_gem_obj_ggtt_offset(obj) + args->offset; |
663 | 670 | ||
664 | while (remain > 0) { |
671 | while (remain > 0) { |
Line 784... | Line 791... | ||
784 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
791 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
785 | /* If we're not in the cpu write domain, set ourself into the gtt |
792 | /* If we're not in the cpu write domain, set ourself into the gtt |
786 | * write domain and manually flush cachelines (if required). This |
793 | * write domain and manually flush cachelines (if required). This |
787 | * optimizes for the case when the gpu will use the data |
794 | * optimizes for the case when the gpu will use the data |
788 | * right away and we therefore have to clflush anyway. */ |
795 | * right away and we therefore have to clflush anyway. */ |
789 | if (obj->cache_level == I915_CACHE_NONE) |
- | |
790 | needs_clflush_after = 1; |
796 | needs_clflush_after = cpu_write_needs_clflush(obj); |
791 | if (obj->gtt_space) { |
797 | if (i915_gem_obj_bound_any(obj)) { |
792 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
798 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
793 | if (ret) |
799 | if (ret) |
794 | return ret; |
800 | return ret; |
795 | } |
801 | } |
796 | } |
802 | } |
797 | /* Same trick applies for invalidate partially written cachelines before |
803 | /* Same trick applies to invalidate partially written cachelines read |
798 | * writing. */ |
804 | * before writing. */ |
799 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) |
805 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) |
800 | && obj->cache_level == I915_CACHE_NONE) |
- | |
801 | needs_clflush_before = 1; |
806 | needs_clflush_before = |
- | 807 | !cpu_cache_is_coherent(dev, obj->cache_level); |
|
Line 802... | Line 808... | ||
802 | 808 | ||
803 | ret = i915_gem_object_get_pages(obj); |
809 | ret = i915_gem_object_get_pages(obj); |
804 | if (ret) |
810 | if (ret) |
Line 875... | Line 881... | ||
875 | * cachelines in-line while writing and the object moved |
881 | * cachelines in-line while writing and the object moved |
876 | * out of the cpu write domain while we've dropped the lock. |
882 | * out of the cpu write domain while we've dropped the lock. |
877 | */ |
883 | */ |
878 | if (!needs_clflush_after && |
884 | if (!needs_clflush_after && |
879 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
885 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
880 | i915_gem_clflush_object(obj); |
886 | if (i915_gem_clflush_object(obj, obj->pin_display)) |
881 | i915_gem_chipset_flush(dev); |
887 | i915_gem_chipset_flush(dev); |
882 | } |
888 | } |
883 | } |
889 | } |
Line 884... | Line 890... | ||
884 | 890 | ||
Line 899... | Line 905... | ||
899 | { |
905 | { |
900 | struct drm_i915_gem_pwrite *args = data; |
906 | struct drm_i915_gem_pwrite *args = data; |
901 | struct drm_i915_gem_object *obj; |
907 | struct drm_i915_gem_object *obj; |
902 | int ret; |
908 | int ret; |
Line -... | Line 909... | ||
- | 909 | ||
- | 910 | if (args->size == 0) |
|
- | 911 | return 0; |
|
903 | 912 | ||
904 | if(args->handle == -2) |
913 | if(args->handle == -2) |
905 | { |
914 | { |
906 | printf("%s handle %d\n", __FUNCTION__, args->handle); |
915 | printf("%s handle %d\n", __FUNCTION__, args->handle); |
907 | return 0; |
916 | return 0; |
Line 908... | Line -... | ||
908 | } |
- | |
909 | - | ||
910 | if (args->size == 0) |
- | |
911 | return 0; |
917 | } |
912 | 918 | ||
913 | ret = i915_mutex_lock_interruptible(dev); |
919 | ret = i915_mutex_lock_interruptible(dev); |
Line 914... | Line 920... | ||
914 | if (ret) |
920 | if (ret) |
Line 947... | Line 953... | ||
947 | // if (obj->phys_obj) { |
953 | // if (obj->phys_obj) { |
948 | // ret = i915_gem_phys_pwrite(dev, obj, args, file); |
954 | // ret = i915_gem_phys_pwrite(dev, obj, args, file); |
949 | // goto out; |
955 | // goto out; |
950 | // } |
956 | // } |
Line 951... | Line 957... | ||
951 | 957 | ||
952 | if (obj->cache_level == I915_CACHE_NONE && |
958 | if (obj->tiling_mode == I915_TILING_NONE && |
953 | obj->tiling_mode == I915_TILING_NONE && |
959 | obj->base.write_domain != I915_GEM_DOMAIN_CPU && |
954 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
960 | cpu_write_needs_clflush(obj)) { |
955 | ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); |
961 | ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); |
956 | /* Note that the gtt paths might fail with non-page-backed user |
962 | /* Note that the gtt paths might fail with non-page-backed user |
957 | * pointers (e.g. gtt mappings when moving data between |
963 | * pointers (e.g. gtt mappings when moving data between |
958 | * textures). Fallback to the shmem path in that case. */ |
964 | * textures). Fallback to the shmem path in that case. */ |
Line 999... | Line 1005... | ||
999 | 1005 | ||
Line 1000... | Line 1006... | ||
1000 | BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); |
1006 | BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); |
1001 | 1007 | ||
1002 | ret = 0; |
1008 | ret = 0; |
Line 1003... | Line 1009... | ||
1003 | if (seqno == ring->outstanding_lazy_request) |
1009 | if (seqno == ring->outstanding_lazy_request) |
1004 | ret = i915_add_request(ring, NULL, NULL); |
1010 | ret = i915_add_request(ring, NULL); |
Line 1005... | Line 1011... | ||
1005 | 1011 | ||
Line 1033... | Line 1039... | ||
1033 | unsigned long timeout_jiffies; |
1039 | unsigned long timeout_jiffies; |
1034 | long end; |
1040 | long end; |
1035 | bool wait_forever = true; |
1041 | bool wait_forever = true; |
1036 | int ret; |
1042 | int ret; |
Line -... | Line 1043... | ||
- | 1043 | ||
- | 1044 | WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n"); |
|
1037 | 1045 | ||
1038 | if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) |
1046 | if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) |
Line 1039... | Line 1047... | ||
1039 | return 0; |
1047 | return 0; |
Line 1040... | Line 1048... | ||
1040 | 1048 | ||
1041 | trace_i915_gem_request_wait_begin(ring, seqno); |
1049 | trace_i915_gem_request_wait_begin(ring, seqno); |
1042 | 1050 | ||
1043 | if (timeout != NULL) { |
1051 | if (timeout != NULL) { |
Line 1044... | Line 1052... | ||
1044 | wait_time = *timeout; |
1052 | wait_time = *timeout; |
Line 1045... | Line 1053... | ||
1045 | wait_forever = false; |
1053 | wait_forever = false; |
1046 | } |
1054 | } |
Line 1047... | Line 1055... | ||
1047 | 1055 | ||
Line 1128... | Line 1136... | ||
1128 | return __wait_seqno(ring, seqno, |
1136 | return __wait_seqno(ring, seqno, |
1129 | atomic_read(&dev_priv->gpu_error.reset_counter), |
1137 | atomic_read(&dev_priv->gpu_error.reset_counter), |
1130 | interruptible, NULL); |
1138 | interruptible, NULL); |
1131 | } |
1139 | } |
Line -... | Line 1140... | ||
- | 1140 | ||
- | 1141 | static int |
|
- | 1142 | i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj, |
|
- | 1143 | struct intel_ring_buffer *ring) |
|
- | 1144 | { |
|
- | 1145 | i915_gem_retire_requests_ring(ring); |
|
- | 1146 | ||
- | 1147 | /* Manually manage the write flush as we may have not yet |
|
- | 1148 | * retired the buffer. |
|
- | 1149 | * |
|
- | 1150 | * Note that the last_write_seqno is always the earlier of |
|
- | 1151 | * the two (read/write) seqno, so if we haved successfully waited, |
|
- | 1152 | * we know we have passed the last write. |
|
- | 1153 | */ |
|
- | 1154 | obj->last_write_seqno = 0; |
|
- | 1155 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
|
- | 1156 | ||
- | 1157 | return 0; |
|
- | 1158 | } |
|
1132 | 1159 | ||
1133 | /** |
1160 | /** |
1134 | * Ensures that all rendering to the object has completed and the object is |
1161 | * Ensures that all rendering to the object has completed and the object is |
1135 | * safe to unbind from the GTT or access from the CPU. |
1162 | * safe to unbind from the GTT or access from the CPU. |
1136 | */ |
1163 | */ |
Line 1148... | Line 1175... | ||
1148 | 1175 | ||
1149 | ret = i915_wait_seqno(ring, seqno); |
1176 | ret = i915_wait_seqno(ring, seqno); |
1150 | if (ret) |
1177 | if (ret) |
Line 1151... | Line 1178... | ||
1151 | return ret; |
1178 | return ret; |
1152 | - | ||
1153 | i915_gem_retire_requests_ring(ring); |
- | |
1154 | - | ||
1155 | /* Manually manage the write flush as we may have not yet |
- | |
1156 | * retired the buffer. |
- | |
1157 | */ |
- | |
1158 | if (obj->last_write_seqno && |
- | |
1159 | i915_seqno_passed(seqno, obj->last_write_seqno)) { |
- | |
1160 | obj->last_write_seqno = 0; |
- | |
1161 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
- | |
1162 | } |
- | |
1163 | 1179 | ||
Line 1164... | Line 1180... | ||
1164 | return 0; |
1180 | return i915_gem_object_wait_rendering__tail(obj, ring); |
1165 | } |
1181 | } |
1166 | 1182 | ||
Line 1195... | Line 1211... | ||
1195 | 1211 | ||
1196 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
1212 | reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); |
1197 | mutex_unlock(&dev->struct_mutex); |
1213 | mutex_unlock(&dev->struct_mutex); |
1198 | ret = __wait_seqno(ring, seqno, reset_counter, true, NULL); |
1214 | ret = __wait_seqno(ring, seqno, reset_counter, true, NULL); |
1199 | mutex_lock(&dev->struct_mutex); |
- | |
1200 | - | ||
1201 | i915_gem_retire_requests_ring(ring); |
- | |
1202 | - | ||
1203 | /* Manually manage the write flush as we may have not yet |
- | |
1204 | * retired the buffer. |
1215 | mutex_lock(&dev->struct_mutex); |
1205 | */ |
- | |
1206 | if (obj->last_write_seqno && |
- | |
1207 | i915_seqno_passed(seqno, obj->last_write_seqno)) { |
- | |
1208 | obj->last_write_seqno = 0; |
- | |
1209 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
- | |
1210 | } |
- | |
1211 | 1216 | if (ret) |
|
- | 1217 | return ret; |
|
- | 1218 | ||
1212 | return ret; |
1219 | return i915_gem_object_wait_rendering__tail(obj, ring); |
Line 1213... | Line 1220... | ||
1213 | } |
1220 | } |
1214 | 1221 | ||
1215 | /** |
1222 | /** |
Line 1366... | Line 1373... | ||
1366 | i915_gem_release_mmap(struct drm_i915_gem_object *obj) |
1373 | i915_gem_release_mmap(struct drm_i915_gem_object *obj) |
1367 | { |
1374 | { |
1368 | if (!obj->fault_mappable) |
1375 | if (!obj->fault_mappable) |
1369 | return; |
1376 | return; |
Line 1370... | Line -... | ||
1370 | - | ||
1371 | if (obj->base.dev->dev_mapping) |
1377 | |
1372 | // unmap_mapping_range(obj->base.dev->dev_mapping, |
- | |
1373 | // (loff_t)obj->base.map_list.hash.key< |
- | |
1374 | // obj->base.size, 1); |
- | |
1375 | 1378 | // drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping); |
|
1376 | obj->fault_mappable = false; |
1379 | obj->fault_mappable = false; |
Line 1377... | Line 1380... | ||
1377 | } |
1380 | } |
1378 | 1381 | ||
Line 1483... | Line 1486... | ||
1483 | DRM_ERROR("Attempting to mmap a purgeable buffer\n"); |
1486 | DRM_ERROR("Attempting to mmap a purgeable buffer\n"); |
1484 | ret = -EINVAL; |
1487 | ret = -EINVAL; |
1485 | goto out; |
1488 | goto out; |
1486 | } |
1489 | } |
1487 | /* Now bind it into the GTT if needed */ |
1490 | /* Now bind it into the GTT if needed */ |
1488 | ret = i915_gem_object_pin(obj, 0, true, false); |
1491 | ret = i915_gem_obj_ggtt_pin(obj, 0, true, false); |
1489 | if (ret) |
1492 | if (ret) |
1490 | goto out; |
1493 | goto out; |
Line 1491... | Line 1494... | ||
1491 | 1494 | ||
1492 | ret = i915_gem_object_set_to_gtt_domain(obj, 1); |
1495 | ret = i915_gem_object_set_to_gtt_domain(obj, 1); |
Line 1497... | Line 1500... | ||
1497 | if (ret) |
1500 | if (ret) |
1498 | goto unpin; |
1501 | goto unpin; |
Line 1499... | Line 1502... | ||
1499 | 1502 | ||
Line 1500... | Line 1503... | ||
1500 | obj->fault_mappable = true; |
1503 | obj->fault_mappable = true; |
Line 1501... | Line 1504... | ||
1501 | 1504 | ||
Line 1502... | Line 1505... | ||
1502 | pfn = dev_priv->gtt.mappable_base + obj->gtt_offset; |
1505 | pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj); |
1503 | 1506 | ||
Line 1515... | Line 1518... | ||
1515 | 1518 | ||
1516 | unpin: |
1519 | unpin: |
Line 1517... | Line 1520... | ||
1517 | i915_gem_object_unpin(obj); |
1520 | i915_gem_object_unpin(obj); |
Line 1518... | Line 1521... | ||
1518 | 1521 | ||
1519 | 1522 | ||
1520 | *offset = (u64)mem; |
1523 | *offset = mem; |
1521 | 1524 | ||
Line 1590... | Line 1593... | ||
1590 | if (ret) { |
1593 | if (ret) { |
1591 | /* In the event of a disaster, abandon all caches and |
1594 | /* In the event of a disaster, abandon all caches and |
1592 | * hope for the best. |
1595 | * hope for the best. |
1593 | */ |
1596 | */ |
1594 | WARN_ON(ret != -EIO); |
1597 | WARN_ON(ret != -EIO); |
1595 | i915_gem_clflush_object(obj); |
1598 | i915_gem_clflush_object(obj, true); |
1596 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1599 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1597 | } |
1600 | } |
Line 1598... | Line 1601... | ||
1598 | 1601 | ||
1599 | if (obj->madv == I915_MADV_DONTNEED) |
1602 | if (obj->madv == I915_MADV_DONTNEED) |
Line 1618... | Line 1621... | ||
1618 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
1621 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
Line 1619... | Line 1622... | ||
1619 | 1622 | ||
1620 | if (obj->pages == NULL) |
1623 | if (obj->pages == NULL) |
Line 1621... | Line -... | ||
1621 | return 0; |
- | |
1622 | - | ||
1623 | BUG_ON(obj->gtt_space); |
1624 | return 0; |
1624 | 1625 | ||
Line -... | Line 1626... | ||
- | 1626 | if (obj->pages_pin_count) |
|
- | 1627 | return -EBUSY; |
|
1625 | if (obj->pages_pin_count) |
1628 | |
1626 | return -EBUSY; |
1629 | BUG_ON(i915_gem_obj_bound_any(obj)); |
1627 | 1630 | ||
1628 | /* ->put_pages might need to allocate memory for the bit17 swizzle |
1631 | /* ->put_pages might need to allocate memory for the bit17 swizzle |
Line 1629... | Line 1632... | ||
1629 | * array, hence protect them from being reaped by removing them from gtt |
1632 | * array, hence protect them from being reaped by removing them from gtt |
1630 | * lists early. */ |
1633 | * lists early. */ |
Line 1631... | Line 1634... | ||
1631 | list_del(&obj->gtt_list); |
1634 | list_del(&obj->global_list); |
Line 1669... | Line 1672... | ||
1669 | if (st == NULL) |
1672 | if (st == NULL) |
1670 | return -ENOMEM; |
1673 | return -ENOMEM; |
Line 1671... | Line 1674... | ||
1671 | 1674 | ||
1672 | page_count = obj->base.size / PAGE_SIZE; |
1675 | page_count = obj->base.size / PAGE_SIZE; |
1673 | if (sg_alloc_table(st, page_count, GFP_KERNEL)) { |
- | |
1674 | sg_free_table(st); |
1676 | if (sg_alloc_table(st, page_count, GFP_KERNEL)) { |
1675 | kfree(st); |
1677 | kfree(st); |
1676 | FAIL(); |
1678 | FAIL(); |
1677 | return -ENOMEM; |
1679 | return -ENOMEM; |
Line 1739... | Line 1741... | ||
1739 | 1741 | ||
1740 | ret = ops->get_pages(obj); |
1742 | ret = ops->get_pages(obj); |
1741 | if (ret) |
1743 | if (ret) |
Line 1742... | Line 1744... | ||
1742 | return ret; |
1744 | return ret; |
1743 | 1745 | ||
1744 | list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
1746 | list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); |
Line 1745... | Line 1747... | ||
1745 | return 0; |
1747 | return 0; |
1746 | } |
1748 | } |
Line 1752... | Line 1754... | ||
1752 | struct drm_device *dev = obj->base.dev; |
1754 | struct drm_device *dev = obj->base.dev; |
1753 | struct drm_i915_private *dev_priv = dev->dev_private; |
1755 | struct drm_i915_private *dev_priv = dev->dev_private; |
1754 | u32 seqno = intel_ring_get_seqno(ring); |
1756 | u32 seqno = intel_ring_get_seqno(ring); |
Line 1755... | Line 1757... | ||
1755 | 1757 | ||
- | 1758 | BUG_ON(ring == NULL); |
|
- | 1759 | if (obj->ring != ring && obj->last_write_seqno) { |
|
- | 1760 | /* Keep the seqno relative to the current ring */ |
|
- | 1761 | obj->last_write_seqno = seqno; |
|
1756 | BUG_ON(ring == NULL); |
1762 | } |
Line 1757... | Line 1763... | ||
1757 | obj->ring = ring; |
1763 | obj->ring = ring; |
1758 | 1764 | ||
1759 | /* Add a reference if we're newly entering the active list. */ |
1765 | /* Add a reference if we're newly entering the active list. */ |
1760 | if (!obj->active) { |
1766 | if (!obj->active) { |
1761 | drm_gem_object_reference(&obj->base); |
1767 | drm_gem_object_reference(&obj->base); |
Line 1762... | Line -... | ||
1762 | obj->active = 1; |
- | |
1763 | } |
- | |
1764 | 1768 | obj->active = 1; |
|
Line 1765... | Line 1769... | ||
1765 | /* Move from whatever list we were on to the tail of execution. */ |
1769 | } |
Line 1766... | Line 1770... | ||
1766 | list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); |
1770 | |
Line 1783... | Line 1787... | ||
1783 | } |
1787 | } |
Line 1784... | Line 1788... | ||
1784 | 1788 | ||
1785 | static void |
1789 | static void |
1786 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
1790 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
1787 | { |
1791 | { |
1788 | struct drm_device *dev = obj->base.dev; |
1792 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
- | 1793 | struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; |
|
Line 1789... | Line 1794... | ||
1789 | struct drm_i915_private *dev_priv = dev->dev_private; |
1794 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); |
1790 | 1795 | ||
Line 1791... | Line 1796... | ||
1791 | BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); |
1796 | BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); |
Line 1792... | Line 1797... | ||
1792 | BUG_ON(!obj->active); |
1797 | BUG_ON(!obj->active); |
1793 | 1798 | ||
Line 1794... | Line 1799... | ||
1794 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
1799 | list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list); |
Line 1877... | Line 1882... | ||
1877 | 1882 | ||
1878 | *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; |
1883 | *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; |
1879 | return 0; |
1884 | return 0; |
Line 1880... | Line -... | ||
1880 | } |
- | |
1881 | 1885 | } |
|
1882 | int |
1886 | |
- | 1887 | int __i915_add_request(struct intel_ring_buffer *ring, |
|
1883 | i915_add_request(struct intel_ring_buffer *ring, |
1888 | struct drm_file *file, |
1884 | struct drm_file *file, |
1889 | struct drm_i915_gem_object *obj, |
1885 | u32 *out_seqno) |
1890 | u32 *out_seqno) |
1886 | { |
1891 | { |
1887 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1892 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1888 | struct drm_i915_gem_request *request; |
1893 | struct drm_i915_gem_request *request; |
1889 | u32 request_ring_position; |
1894 | u32 request_ring_position, request_start; |
Line -... | Line 1895... | ||
- | 1895 | int was_empty; |
|
1890 | int was_empty; |
1896 | int ret; |
1891 | int ret; |
1897 | |
1892 | 1898 | request_start = intel_ring_get_tail(ring); |
|
1893 | /* |
1899 | /* |
1894 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
1900 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
Line 1919... | Line 1925... | ||
1919 | return ret; |
1925 | return ret; |
1920 | } |
1926 | } |
Line 1921... | Line 1927... | ||
1921 | 1927 | ||
1922 | request->seqno = intel_ring_get_seqno(ring); |
1928 | request->seqno = intel_ring_get_seqno(ring); |
- | 1929 | request->ring = ring; |
|
1923 | request->ring = ring; |
1930 | request->head = request_start; |
- | 1931 | request->tail = request_ring_position; |
|
- | 1932 | request->ctx = ring->last_context; |
|
- | 1933 | request->batch_obj = obj; |
|
- | 1934 | ||
- | 1935 | /* Whilst this request exists, batch_obj will be on the |
|
- | 1936 | * active_list, and so will hold the active reference. Only when this |
|
- | 1937 | * request is retired will the the batch_obj be moved onto the |
|
- | 1938 | * inactive_list and lose its active reference. Hence we do not need |
|
- | 1939 | * to explicitly hold another reference here. |
|
- | 1940 | */ |
|
- | 1941 | ||
- | 1942 | if (request->ctx) |
|
- | 1943 | i915_gem_context_reference(request->ctx); |
|
1924 | request->tail = request_ring_position; |
1944 | |
1925 | request->emitted_jiffies = GetTimerTicks(); |
1945 | request->emitted_jiffies = GetTimerTicks(); |
1926 | was_empty = list_empty(&ring->request_list); |
1946 | was_empty = list_empty(&ring->request_list); |
1927 | list_add_tail(&request->list, &ring->request_list); |
1947 | list_add_tail(&request->list, &ring->request_list); |
Line 1938... | Line 1958... | ||
1938 | } |
1958 | } |
Line 1939... | Line 1959... | ||
1939 | 1959 | ||
1940 | trace_i915_gem_request_add(ring, request->seqno); |
1960 | trace_i915_gem_request_add(ring, request->seqno); |
Line 1941... | Line 1961... | ||
1941 | ring->outstanding_lazy_request = 0; |
1961 | ring->outstanding_lazy_request = 0; |
1942 | 1962 | ||
1943 | if (!dev_priv->mm.suspended) { |
- | |
1944 | if (i915_enable_hangcheck) { |
- | |
1945 | // mod_timer(&dev_priv->hangcheck_timer, |
- | |
1946 | // jiffies + |
1963 | if (!dev_priv->ums.mm_suspended) { |
1947 | // msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); |
1964 | // i915_queue_hangcheck(ring->dev); |
1948 | } |
1965 | |
1949 | if (was_empty) { |
1966 | if (was_empty) { |
1950 | queue_delayed_work(dev_priv->wq, |
1967 | queue_delayed_work(dev_priv->wq, |
1951 | &dev_priv->mm.retire_work, |
1968 | &dev_priv->mm.retire_work, |
Line 1973... | Line 1990... | ||
1973 | request->file_priv = NULL; |
1990 | request->file_priv = NULL; |
1974 | } |
1991 | } |
1975 | spin_unlock(&file_priv->mm.lock); |
1992 | spin_unlock(&file_priv->mm.lock); |
1976 | } |
1993 | } |
Line -... | Line 1994... | ||
- | 1994 | ||
- | 1995 | static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj, |
|
- | 1996 | struct i915_address_space *vm) |
|
- | 1997 | { |
|
- | 1998 | if (acthd >= i915_gem_obj_offset(obj, vm) && |
|
- | 1999 | acthd < i915_gem_obj_offset(obj, vm) + obj->base.size) |
|
- | 2000 | return true; |
|
- | 2001 | ||
- | 2002 | return false; |
|
- | 2003 | } |
|
- | 2004 | ||
- | 2005 | static bool i915_head_inside_request(const u32 acthd_unmasked, |
|
- | 2006 | const u32 request_start, |
|
- | 2007 | const u32 request_end) |
|
- | 2008 | { |
|
- | 2009 | const u32 acthd = acthd_unmasked & HEAD_ADDR; |
|
- | 2010 | ||
- | 2011 | if (request_start < request_end) { |
|
- | 2012 | if (acthd >= request_start && acthd < request_end) |
|
- | 2013 | return true; |
|
- | 2014 | } else if (request_start > request_end) { |
|
- | 2015 | if (acthd >= request_start || acthd < request_end) |
|
- | 2016 | return true; |
|
- | 2017 | } |
|
- | 2018 | ||
- | 2019 | return false; |
|
- | 2020 | } |
|
- | 2021 | ||
- | 2022 | static struct i915_address_space * |
|
- | 2023 | request_to_vm(struct drm_i915_gem_request *request) |
|
- | 2024 | { |
|
- | 2025 | struct drm_i915_private *dev_priv = request->ring->dev->dev_private; |
|
- | 2026 | struct i915_address_space *vm; |
|
- | 2027 | ||
- | 2028 | vm = &dev_priv->gtt.base; |
|
- | 2029 | ||
- | 2030 | return vm; |
|
- | 2031 | } |
|
- | 2032 | ||
- | 2033 | static bool i915_request_guilty(struct drm_i915_gem_request *request, |
|
- | 2034 | const u32 acthd, bool *inside) |
|
- | 2035 | { |
|
- | 2036 | /* There is a possibility that unmasked head address |
|
- | 2037 | * pointing inside the ring, matches the batch_obj address range. |
|
- | 2038 | * However this is extremely unlikely. |
|
- | 2039 | */ |
|
- | 2040 | if (request->batch_obj) { |
|
- | 2041 | if (i915_head_inside_object(acthd, request->batch_obj, |
|
- | 2042 | request_to_vm(request))) { |
|
- | 2043 | *inside = true; |
|
- | 2044 | return true; |
|
- | 2045 | } |
|
- | 2046 | } |
|
- | 2047 | ||
- | 2048 | if (i915_head_inside_request(acthd, request->head, request->tail)) { |
|
- | 2049 | *inside = false; |
|
- | 2050 | return true; |
|
- | 2051 | } |
|
- | 2052 | ||
- | 2053 | return false; |
|
- | 2054 | } |
|
- | 2055 | ||
- | 2056 | static void i915_set_reset_status(struct intel_ring_buffer *ring, |
|
- | 2057 | struct drm_i915_gem_request *request, |
|
- | 2058 | u32 acthd) |
|
- | 2059 | { |
|
- | 2060 | struct i915_ctx_hang_stats *hs = NULL; |
|
- | 2061 | bool inside, guilty; |
|
- | 2062 | unsigned long offset = 0; |
|
- | 2063 | ||
- | 2064 | /* Innocent until proven guilty */ |
|
- | 2065 | guilty = false; |
|
- | 2066 | ||
- | 2067 | if (request->batch_obj) |
|
- | 2068 | offset = i915_gem_obj_offset(request->batch_obj, |
|
- | 2069 | request_to_vm(request)); |
|
- | 2070 | ||
- | 2071 | if (ring->hangcheck.action != HANGCHECK_WAIT && |
|
- | 2072 | i915_request_guilty(request, acthd, &inside)) { |
|
- | 2073 | DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n", |
|
- | 2074 | ring->name, |
|
- | 2075 | inside ? "inside" : "flushing", |
|
- | 2076 | offset, |
|
- | 2077 | request->ctx ? request->ctx->id : 0, |
|
- | 2078 | acthd); |
|
- | 2079 | ||
- | 2080 | guilty = true; |
|
- | 2081 | } |
|
- | 2082 | ||
- | 2083 | /* If contexts are disabled or this is the default context, use |
|
- | 2084 | * file_priv->reset_state |
|
- | 2085 | */ |
|
- | 2086 | if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID) |
|
- | 2087 | hs = &request->ctx->hang_stats; |
|
- | 2088 | else if (request->file_priv) |
|
- | 2089 | hs = &request->file_priv->hang_stats; |
|
- | 2090 | ||
- | 2091 | if (hs) { |
|
- | 2092 | if (guilty) |
|
- | 2093 | hs->batch_active++; |
|
- | 2094 | else |
|
- | 2095 | hs->batch_pending++; |
|
- | 2096 | } |
|
- | 2097 | } |
|
- | 2098 | ||
- | 2099 | static void i915_gem_free_request(struct drm_i915_gem_request *request) |
|
- | 2100 | { |
|
- | 2101 | list_del(&request->list); |
|
- | 2102 | i915_gem_request_remove_from_client(request); |
|
- | 2103 | ||
- | 2104 | if (request->ctx) |
|
- | 2105 | i915_gem_context_unreference(request->ctx); |
|
- | 2106 | ||
- | 2107 | kfree(request); |
|
- | 2108 | } |
|
1977 | 2109 | ||
1978 | static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, |
2110 | static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, |
1979 | struct intel_ring_buffer *ring) |
2111 | struct intel_ring_buffer *ring) |
- | 2112 | { |
|
- | 2113 | u32 completed_seqno; |
|
- | 2114 | u32 acthd; |
|
- | 2115 | ||
- | 2116 | acthd = intel_ring_get_active_head(ring); |
|
- | 2117 | completed_seqno = ring->get_seqno(ring, false); |
|
1980 | { |
2118 | |
1981 | while (!list_empty(&ring->request_list)) { |
2119 | while (!list_empty(&ring->request_list)) { |
Line 1982... | Line 2120... | ||
1982 | struct drm_i915_gem_request *request; |
2120 | struct drm_i915_gem_request *request; |
1983 | 2121 | ||
1984 | request = list_first_entry(&ring->request_list, |
2122 | request = list_first_entry(&ring->request_list, |
Line 1985... | Line 2123... | ||
1985 | struct drm_i915_gem_request, |
2123 | struct drm_i915_gem_request, |
1986 | list); |
2124 | list); |
- | 2125 | ||
1987 | 2126 | if (request->seqno > completed_seqno) |
|
1988 | list_del(&request->list); |
2127 | i915_set_reset_status(ring, request, acthd); |
Line 1989... | Line 2128... | ||
1989 | i915_gem_request_remove_from_client(request); |
2128 | |
1990 | kfree(request); |
2129 | i915_gem_free_request(request); |
Line 2006... | Line 2145... | ||
2006 | struct drm_i915_private *dev_priv = dev->dev_private; |
2145 | struct drm_i915_private *dev_priv = dev->dev_private; |
2007 | int i; |
2146 | int i; |
Line 2008... | Line 2147... | ||
2008 | 2147 | ||
2009 | for (i = 0; i < dev_priv->num_fence_regs; i++) { |
2148 | for (i = 0; i < dev_priv->num_fence_regs; i++) { |
- | 2149 | struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
|
- | 2150 | ||
- | 2151 | /* |
|
- | 2152 | * Commit delayed tiling changes if we have an object still |
|
- | 2153 | * attached to the fence, otherwise just clear the fence. |
|
- | 2154 | */ |
|
- | 2155 | if (reg->obj) { |
|
- | 2156 | i915_gem_object_update_fence(reg->obj, reg, |
|
- | 2157 | reg->obj->tiling_mode); |
|
2010 | struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
2158 | } else { |
- | 2159 | i915_gem_write_fence(dev, i, NULL); |
|
2011 | i915_gem_write_fence(dev, i, reg->obj); |
2160 | } |
2012 | } |
2161 | } |
Line 2013... | Line 2162... | ||
2013 | } |
2162 | } |
2014 | 2163 | ||
2015 | void i915_gem_reset(struct drm_device *dev) |
2164 | void i915_gem_reset(struct drm_device *dev) |
2016 | { |
- | |
2017 | struct drm_i915_private *dev_priv = dev->dev_private; |
2165 | { |
2018 | struct drm_i915_gem_object *obj; |
2166 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 2019... | Line 2167... | ||
2019 | struct intel_ring_buffer *ring; |
2167 | struct intel_ring_buffer *ring; |
2020 | int i; |
2168 | int i; |
Line 2021... | Line -... | ||
2021 | - | ||
2022 | for_each_ring(ring, dev_priv, i) |
- | |
2023 | i915_gem_reset_ring_lists(dev_priv, ring); |
- | |
2024 | - | ||
2025 | /* Move everything out of the GPU domains to ensure we do any |
- | |
2026 | * necessary invalidation upon reuse. |
- | |
2027 | */ |
- | |
2028 | list_for_each_entry(obj, |
- | |
2029 | &dev_priv->mm.inactive_list, |
- | |
2030 | mm_list) |
- | |
2031 | { |
2169 | |
2032 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
2170 | for_each_ring(ring, dev_priv, i) |
Line 2033... | Line 2171... | ||
2033 | } |
2171 | i915_gem_reset_ring_lists(dev_priv, ring); |
2034 | 2172 | ||
Line 2066... | Line 2204... | ||
2066 | * of tail of the request to update the last known position |
2204 | * of tail of the request to update the last known position |
2067 | * of the GPU head. |
2205 | * of the GPU head. |
2068 | */ |
2206 | */ |
2069 | ring->last_retired_head = request->tail; |
2207 | ring->last_retired_head = request->tail; |
Line 2070... | Line -... | ||
2070 | - | ||
2071 | list_del(&request->list); |
2208 | |
2072 | i915_gem_request_remove_from_client(request); |
- | |
2073 | kfree(request); |
2209 | i915_gem_free_request(request); |
Line 2074... | Line 2210... | ||
2074 | } |
2210 | } |
2075 | 2211 | ||
2076 | /* Move any buffers on the active list that are no longer referenced |
2212 | /* Move any buffers on the active list that are no longer referenced |
Line 2135... | Line 2271... | ||
2135 | * objects indefinitely. |
2271 | * objects indefinitely. |
2136 | */ |
2272 | */ |
2137 | idle = true; |
2273 | idle = true; |
2138 | for_each_ring(ring, dev_priv, i) { |
2274 | for_each_ring(ring, dev_priv, i) { |
2139 | if (ring->gpu_caches_dirty) |
2275 | if (ring->gpu_caches_dirty) |
2140 | i915_add_request(ring, NULL, NULL); |
2276 | i915_add_request(ring, NULL); |
Line 2141... | Line 2277... | ||
2141 | 2277 | ||
2142 | idle &= list_empty(&ring->request_list); |
2278 | idle &= list_empty(&ring->request_list); |
Line 2143... | Line 2279... | ||
2143 | } |
2279 | } |
2144 | 2280 | ||
2145 | if (!dev_priv->mm.suspended && !idle) |
2281 | if (!dev_priv->ums.mm_suspended && !idle) |
2146 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, |
2282 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, |
2147 | round_jiffies_up_relative(HZ)); |
2283 | round_jiffies_up_relative(HZ)); |
Line 2275... | Line 2411... | ||
2275 | trace_i915_gem_object_change_domain(obj, |
2411 | trace_i915_gem_object_change_domain(obj, |
2276 | old_read_domains, |
2412 | old_read_domains, |
2277 | old_write_domain); |
2413 | old_write_domain); |
2278 | } |
2414 | } |
Line 2279... | Line -... | ||
2279 | - | ||
2280 | /** |
- | |
2281 | * Unbinds an object from the GTT aperture. |
- | |
2282 | */ |
- | |
2283 | int |
2415 | |
2284 | i915_gem_object_unbind(struct drm_i915_gem_object *obj) |
2416 | int i915_vma_unbind(struct i915_vma *vma) |
- | 2417 | { |
|
2285 | { |
2418 | struct drm_i915_gem_object *obj = vma->obj; |
2286 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
2419 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
Line 2287... | Line 2420... | ||
2287 | int ret; |
2420 | int ret; |
2288 | 2421 | ||
Line 2289... | Line 2422... | ||
2289 | if(obj == get_fb_obj()) |
2422 | if(obj == get_fb_obj()) |
2290 | return 0; |
2423 | return 0; |
Line -... | Line 2424... | ||
- | 2424 | ||
- | 2425 | if (list_empty(&vma->vma_link)) |
|
- | 2426 | return 0; |
|
2291 | 2427 | ||
2292 | if (obj->gtt_space == NULL) |
2428 | if (!drm_mm_node_allocated(&vma->node)) |
Line 2293... | Line 2429... | ||
2293 | return 0; |
2429 | goto destroy; |
Line 2310... | Line 2446... | ||
2310 | /* release the fence reg _after_ flushing */ |
2446 | /* release the fence reg _after_ flushing */ |
2311 | ret = i915_gem_object_put_fence(obj); |
2447 | ret = i915_gem_object_put_fence(obj); |
2312 | if (ret) |
2448 | if (ret) |
2313 | return ret; |
2449 | return ret; |
Line 2314... | Line 2450... | ||
2314 | 2450 | ||
Line 2315... | Line 2451... | ||
2315 | trace_i915_gem_object_unbind(obj); |
2451 | trace_i915_vma_unbind(vma); |
2316 | 2452 | ||
2317 | if (obj->has_global_gtt_mapping) |
2453 | if (obj->has_global_gtt_mapping) |
2318 | i915_gem_gtt_unbind_object(obj); |
2454 | i915_gem_gtt_unbind_object(obj); |
2319 | if (obj->has_aliasing_ppgtt_mapping) { |
2455 | if (obj->has_aliasing_ppgtt_mapping) { |
2320 | i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); |
2456 | i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); |
2321 | obj->has_aliasing_ppgtt_mapping = 0; |
2457 | obj->has_aliasing_ppgtt_mapping = 0; |
- | 2458 | } |
|
Line 2322... | Line 2459... | ||
2322 | } |
2459 | i915_gem_gtt_finish_object(obj); |
2323 | i915_gem_gtt_finish_object(obj); |
- | |
2324 | 2460 | i915_gem_object_unpin_pages(obj); |
|
- | 2461 | ||
2325 | list_del(&obj->mm_list); |
2462 | list_del(&vma->mm_list); |
Line 2326... | Line 2463... | ||
2326 | list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
2463 | /* Avoid an unnecessary call to unbind on rebind. */ |
- | 2464 | if (i915_is_ggtt(vma->vm)) |
|
- | 2465 | obj->map_and_fenceable = true; |
|
2327 | /* Avoid an unnecessary call to unbind on rebind. */ |
2466 | |
- | 2467 | drm_mm_remove_node(&vma->node); |
|
- | 2468 | ||
2328 | obj->map_and_fenceable = true; |
2469 | destroy: |
- | 2470 | i915_gem_vma_destroy(vma); |
|
- | 2471 | ||
- | 2472 | /* Since the unbound list is global, only move to that list if |
|
- | 2473 | * no more VMAs exist. |
|
Line 2329... | Line 2474... | ||
2329 | 2474 | * NB: Until we have real VMAs there will only ever be one */ |
|
2330 | drm_mm_put_block(obj->gtt_space); |
2475 | WARN_ON(!list_empty(&obj->vma_list)); |
Line -... | Line 2476... | ||
- | 2476 | if (list_empty(&obj->vma_list)) |
|
- | 2477 | list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); |
|
- | 2478 | ||
- | 2479 | return 0; |
|
- | 2480 | } |
|
- | 2481 | ||
- | 2482 | /** |
|
- | 2483 | * Unbinds an object from the global GTT aperture. |
|
- | 2484 | */ |
|
- | 2485 | int |
|
- | 2486 | i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj) |
|
- | 2487 | { |
|
- | 2488 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
|
- | 2489 | struct i915_address_space *ggtt = &dev_priv->gtt.base; |
|
- | 2490 | ||
- | 2491 | if (!i915_gem_obj_ggtt_bound(obj)) |
|
- | 2492 | return 0; |
|
- | 2493 | ||
- | 2494 | if (obj->pin_count) |
|
- | 2495 | return -EBUSY; |
|
2331 | obj->gtt_space = NULL; |
2496 | |
2332 | obj->gtt_offset = 0; |
2497 | BUG_ON(obj->pages == NULL); |
2333 | 2498 | ||
2334 | return 0; |
2499 | return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt)); |
2335 | } |
2500 | } |
Line 2358... | Line 2523... | ||
2358 | struct drm_i915_gem_object *obj) |
2523 | struct drm_i915_gem_object *obj) |
2359 | { |
2524 | { |
2360 | drm_i915_private_t *dev_priv = dev->dev_private; |
2525 | drm_i915_private_t *dev_priv = dev->dev_private; |
2361 | int fence_reg; |
2526 | int fence_reg; |
2362 | int fence_pitch_shift; |
2527 | int fence_pitch_shift; |
2363 | uint64_t val; |
- | |
Line 2364... | Line 2528... | ||
2364 | 2528 | ||
2365 | if (INTEL_INFO(dev)->gen >= 6) { |
2529 | if (INTEL_INFO(dev)->gen >= 6) { |
2366 | fence_reg = FENCE_REG_SANDYBRIDGE_0; |
2530 | fence_reg = FENCE_REG_SANDYBRIDGE_0; |
2367 | fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT; |
2531 | fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT; |
2368 | } else { |
2532 | } else { |
2369 | fence_reg = FENCE_REG_965_0; |
2533 | fence_reg = FENCE_REG_965_0; |
2370 | fence_pitch_shift = I965_FENCE_PITCH_SHIFT; |
2534 | fence_pitch_shift = I965_FENCE_PITCH_SHIFT; |
Line -... | Line 2535... | ||
- | 2535 | } |
|
- | 2536 | ||
- | 2537 | fence_reg += reg * 8; |
|
- | 2538 | ||
- | 2539 | /* To w/a incoherency with non-atomic 64-bit register updates, |
|
- | 2540 | * we split the 64-bit update into two 32-bit writes. In order |
|
- | 2541 | * for a partial fence not to be evaluated between writes, we |
|
- | 2542 | * precede the update with write to turn off the fence register, |
|
- | 2543 | * and only enable the fence as the last step. |
|
- | 2544 | * |
|
- | 2545 | * For extra levels of paranoia, we make sure each step lands |
|
- | 2546 | * before applying the next step. |
|
- | 2547 | */ |
|
- | 2548 | I915_WRITE(fence_reg, 0); |
|
2371 | } |
2549 | POSTING_READ(fence_reg); |
2372 | 2550 | ||
- | 2551 | if (obj) { |
|
Line 2373... | Line 2552... | ||
2373 | if (obj) { |
2552 | u32 size = i915_gem_obj_ggtt_size(obj); |
2374 | u32 size = obj->gtt_space->size; |
2553 | uint64_t val; |
2375 | 2554 | ||
2376 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
2555 | val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) & |
2377 | 0xfffff000) << 32; |
2556 | 0xfffff000) << 32; |
2378 | val |= obj->gtt_offset & 0xfffff000; |
2557 | val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000; |
2379 | val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; |
2558 | val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift; |
2380 | if (obj->tiling_mode == I915_TILING_Y) |
- | |
2381 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
- | |
Line 2382... | Line 2559... | ||
2382 | val |= I965_FENCE_REG_VALID; |
2559 | if (obj->tiling_mode == I915_TILING_Y) |
- | 2560 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
|
- | 2561 | val |= I965_FENCE_REG_VALID; |
|
2383 | } else |
2562 | |
2384 | val = 0; |
2563 | I915_WRITE(fence_reg + 4, val >> 32); |
- | 2564 | POSTING_READ(fence_reg + 4); |
|
- | 2565 | ||
- | 2566 | I915_WRITE(fence_reg + 0, val); |
|
- | 2567 | POSTING_READ(fence_reg); |
|
2385 | 2568 | } else { |
|
Line 2386... | Line 2569... | ||
2386 | fence_reg += reg * 8; |
2569 | I915_WRITE(fence_reg + 4, 0); |
2387 | I915_WRITE64(fence_reg, val); |
2570 | POSTING_READ(fence_reg + 4); |
2388 | POSTING_READ(fence_reg); |
2571 | } |
2389 | } |
2572 | } |
2390 | 2573 | ||
Line 2391... | Line 2574... | ||
2391 | static void i915_write_fence_reg(struct drm_device *dev, int reg, |
2574 | static void i915_write_fence_reg(struct drm_device *dev, int reg, |
2392 | struct drm_i915_gem_object *obj) |
2575 | struct drm_i915_gem_object *obj) |
2393 | { |
2576 | { |
2394 | drm_i915_private_t *dev_priv = dev->dev_private; |
2577 | drm_i915_private_t *dev_priv = dev->dev_private; |
Line 2395... | Line 2578... | ||
2395 | u32 val; |
2578 | u32 val; |
2396 | 2579 | ||
2397 | if (obj) { |
2580 | if (obj) { |
2398 | u32 size = obj->gtt_space->size; |
2581 | u32 size = i915_gem_obj_ggtt_size(obj); |
2399 | int pitch_val; |
2582 | int pitch_val; |
Line 2400... | Line 2583... | ||
2400 | int tile_width; |
2583 | int tile_width; |
2401 | 2584 | ||
2402 | WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || |
2585 | WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) || |
2403 | (size & -size) != size || |
2586 | (size & -size) != size || |
Line 2404... | Line 2587... | ||
2404 | (obj->gtt_offset & (size - 1)), |
2587 | (i915_gem_obj_ggtt_offset(obj) & (size - 1)), |
2405 | "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", |
2588 | "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", |
2406 | obj->gtt_offset, obj->map_and_fenceable, size); |
2589 | i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size); |
Line 2407... | Line 2590... | ||
2407 | 2590 | ||
2408 | if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) |
2591 | if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) |
2409 | tile_width = 128; |
2592 | tile_width = 128; |
2410 | else |
2593 | else |
2411 | tile_width = 512; |
2594 | tile_width = 512; |
2412 | 2595 | ||
Line 2437... | Line 2620... | ||
2437 | { |
2620 | { |
2438 | drm_i915_private_t *dev_priv = dev->dev_private; |
2621 | drm_i915_private_t *dev_priv = dev->dev_private; |
2439 | uint32_t val; |
2622 | uint32_t val; |
Line 2440... | Line 2623... | ||
2440 | 2623 | ||
2441 | if (obj) { |
2624 | if (obj) { |
2442 | u32 size = obj->gtt_space->size; |
2625 | u32 size = i915_gem_obj_ggtt_size(obj); |
Line 2443... | Line 2626... | ||
2443 | uint32_t pitch_val; |
2626 | uint32_t pitch_val; |
2444 | 2627 | ||
2445 | WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || |
2628 | WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) || |
2446 | (size & -size) != size || |
2629 | (size & -size) != size || |
2447 | (obj->gtt_offset & (size - 1)), |
2630 | (i915_gem_obj_ggtt_offset(obj) & (size - 1)), |
Line 2448... | Line 2631... | ||
2448 | "object 0x%08x not 512K or pot-size 0x%08x aligned\n", |
2631 | "object 0x%08lx not 512K or pot-size 0x%08x aligned\n", |
2449 | obj->gtt_offset, size); |
2632 | i915_gem_obj_ggtt_offset(obj), size); |
Line 2450... | Line 2633... | ||
2450 | 2633 | ||
2451 | pitch_val = obj->stride / 128; |
2634 | pitch_val = obj->stride / 128; |
2452 | pitch_val = ffs(pitch_val) - 1; |
2635 | pitch_val = ffs(pitch_val) - 1; |
2453 | 2636 | ||
2454 | val = obj->gtt_offset; |
2637 | val = i915_gem_obj_ggtt_offset(obj); |
2455 | if (obj->tiling_mode == I915_TILING_Y) |
2638 | if (obj->tiling_mode == I915_TILING_Y) |
Line 2478... | Line 2661... | ||
2478 | * and all writes before removing the fence. |
2661 | * and all writes before removing the fence. |
2479 | */ |
2662 | */ |
2480 | if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) |
2663 | if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj)) |
2481 | mb(); |
2664 | mb(); |
Line -... | Line 2665... | ||
- | 2665 | ||
- | 2666 | WARN(obj && (!obj->stride || !obj->tiling_mode), |
|
- | 2667 | "bogus fence setup with stride: 0x%x, tiling mode: %i\n", |
|
- | 2668 | obj->stride, obj->tiling_mode); |
|
2482 | 2669 | ||
2483 | switch (INTEL_INFO(dev)->gen) { |
2670 | switch (INTEL_INFO(dev)->gen) { |
2484 | case 7: |
2671 | case 7: |
2485 | case 6: |
2672 | case 6: |
2486 | case 5: |
2673 | case 5: |
Line 2501... | Line 2688... | ||
2501 | struct drm_i915_fence_reg *fence) |
2688 | struct drm_i915_fence_reg *fence) |
2502 | { |
2689 | { |
2503 | return fence - dev_priv->fence_regs; |
2690 | return fence - dev_priv->fence_regs; |
2504 | } |
2691 | } |
Line 2505... | Line -... | ||
2505 | - | ||
2506 | static void i915_gem_write_fence__ipi(void *data) |
- | |
2507 | { |
- | |
2508 | asm volatile("wbinvd"); |
- | |
2509 | - | ||
2510 | } |
- | |
2511 | 2692 | ||
2512 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
2693 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
2513 | struct drm_i915_fence_reg *fence, |
2694 | struct drm_i915_fence_reg *fence, |
2514 | bool enable) |
2695 | bool enable) |
2515 | { |
- | |
2516 | struct drm_device *dev = obj->base.dev; |
2696 | { |
2517 | struct drm_i915_private *dev_priv = dev->dev_private; |
2697 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
Line 2518... | Line -... | ||
2518 | int fence_reg = fence_number(dev_priv, fence); |
- | |
2519 | - | ||
2520 | /* In order to fully serialize access to the fenced region and |
- | |
2521 | * the update to the fence register we need to take extreme |
- | |
2522 | * measures on SNB+. In theory, the write to the fence register |
- | |
2523 | * flushes all memory transactions before, and coupled with the |
- | |
2524 | * mb() placed around the register write we serialise all memory |
- | |
2525 | * operations with respect to the changes in the tiler. Yet, on |
- | |
2526 | * SNB+ we need to take a step further and emit an explicit wbinvd() |
- | |
2527 | * on each processor in order to manually flush all memory |
- | |
2528 | * transactions before updating the fence register. |
- | |
2529 | */ |
- | |
2530 | if (HAS_LLC(obj->base.dev)) |
2698 | int reg = fence_number(dev_priv, fence); |
Line 2531... | Line 2699... | ||
2531 | on_each_cpu(i915_gem_write_fence__ipi, NULL, 1); |
2699 | |
2532 | i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL); |
2700 | i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); |
2533 | 2701 | ||
2534 | if (enable) { |
2702 | if (enable) { |
2535 | obj->fence_reg = fence_reg; |
2703 | obj->fence_reg = reg; |
2536 | fence->obj = obj; |
2704 | fence->obj = obj; |
2537 | list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
2705 | list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
2538 | } else { |
2706 | } else { |
2539 | obj->fence_reg = I915_FENCE_REG_NONE; |
2707 | obj->fence_reg = I915_FENCE_REG_NONE; |
- | 2708 | fence->obj = NULL; |
|
2540 | fence->obj = NULL; |
2709 | list_del_init(&fence->lru_list); |
Line 2541... | Line 2710... | ||
2541 | list_del_init(&fence->lru_list); |
2710 | } |
2542 | } |
2711 | obj->fence_dirty = false; |
2543 | } |
2712 | } |
Line 2667... | Line 2836... | ||
2667 | } |
2836 | } |
2668 | } else |
2837 | } else |
2669 | return 0; |
2838 | return 0; |
Line 2670... | Line 2839... | ||
2670 | 2839 | ||
2671 | i915_gem_object_update_fence(obj, reg, enable); |
- | |
Line 2672... | Line 2840... | ||
2672 | obj->fence_dirty = false; |
2840 | i915_gem_object_update_fence(obj, reg, enable); |
2673 | 2841 | ||
Line 2674... | Line 2842... | ||
2674 | return 0; |
2842 | return 0; |
Line 2685... | Line 2853... | ||
2685 | * crossing memory domains and dying. |
2853 | * crossing memory domains and dying. |
2686 | */ |
2854 | */ |
2687 | if (HAS_LLC(dev)) |
2855 | if (HAS_LLC(dev)) |
2688 | return true; |
2856 | return true; |
Line 2689... | Line 2857... | ||
2689 | 2857 | ||
2690 | if (gtt_space == NULL) |
2858 | if (!drm_mm_node_allocated(gtt_space)) |
Line 2691... | Line 2859... | ||
2691 | return true; |
2859 | return true; |
2692 | 2860 | ||
Line 2709... | Line 2877... | ||
2709 | #if WATCH_GTT |
2877 | #if WATCH_GTT |
2710 | struct drm_i915_private *dev_priv = dev->dev_private; |
2878 | struct drm_i915_private *dev_priv = dev->dev_private; |
2711 | struct drm_i915_gem_object *obj; |
2879 | struct drm_i915_gem_object *obj; |
2712 | int err = 0; |
2880 | int err = 0; |
Line 2713... | Line 2881... | ||
2713 | 2881 | ||
2714 | list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { |
2882 | list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) { |
2715 | if (obj->gtt_space == NULL) { |
2883 | if (obj->gtt_space == NULL) { |
2716 | printk(KERN_ERR "object found on GTT list with no space reserved\n"); |
2884 | printk(KERN_ERR "object found on GTT list with no space reserved\n"); |
2717 | err++; |
2885 | err++; |
2718 | continue; |
2886 | continue; |
Line 2719... | Line 2887... | ||
2719 | } |
2887 | } |
2720 | 2888 | ||
2721 | if (obj->cache_level != obj->gtt_space->color) { |
2889 | if (obj->cache_level != obj->gtt_space->color) { |
2722 | printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", |
2890 | printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", |
2723 | obj->gtt_space->start, |
2891 | i915_gem_obj_ggtt_offset(obj), |
2724 | obj->gtt_space->start + obj->gtt_space->size, |
2892 | i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj), |
2725 | obj->cache_level, |
2893 | obj->cache_level, |
2726 | obj->gtt_space->color); |
2894 | obj->gtt_space->color); |
2727 | err++; |
2895 | err++; |
Line 2728... | Line 2896... | ||
2728 | continue; |
2896 | continue; |
2729 | } |
2897 | } |
2730 | 2898 | ||
2731 | if (!i915_gem_valid_gtt_space(dev, |
2899 | if (!i915_gem_valid_gtt_space(dev, |
2732 | obj->gtt_space, |
2900 | obj->gtt_space, |
2733 | obj->cache_level)) { |
2901 | obj->cache_level)) { |
2734 | printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n", |
2902 | printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n", |
2735 | obj->gtt_space->start, |
2903 | i915_gem_obj_ggtt_offset(obj), |
2736 | obj->gtt_space->start + obj->gtt_space->size, |
2904 | i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj), |
2737 | obj->cache_level); |
2905 | obj->cache_level); |
2738 | err++; |
2906 | err++; |
Line 2746... | Line 2914... | ||
2746 | 2914 | ||
2747 | /** |
2915 | /** |
2748 | * Finds free space in the GTT aperture and binds the object there. |
2916 | * Finds free space in the GTT aperture and binds the object there. |
2749 | */ |
2917 | */ |
2750 | static int |
2918 | static int |
- | 2919 | i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, |
|
2751 | i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
2920 | struct i915_address_space *vm, |
2752 | unsigned alignment, |
2921 | unsigned alignment, |
2753 | bool map_and_fenceable, |
2922 | bool map_and_fenceable, |
2754 | bool nonblocking) |
2923 | bool nonblocking) |
2755 | { |
2924 | { |
2756 | struct drm_device *dev = obj->base.dev; |
2925 | struct drm_device *dev = obj->base.dev; |
2757 | drm_i915_private_t *dev_priv = dev->dev_private; |
- | |
2758 | struct drm_mm_node *node; |
2926 | drm_i915_private_t *dev_priv = dev->dev_private; |
- | 2927 | u32 size, fence_size, fence_alignment, unfenced_alignment; |
|
- | 2928 | size_t gtt_max = |
|
2759 | u32 size, fence_size, fence_alignment, unfenced_alignment; |
2929 | map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total; |
2760 | bool mappable, fenceable; |
2930 | struct i915_vma *vma; |
Line 2761... | Line 2931... | ||
2761 | int ret; |
2931 | int ret; |
2762 | 2932 | ||
2763 | fence_size = i915_gem_get_gtt_size(dev, |
2933 | fence_size = i915_gem_get_gtt_size(dev, |
Line 2782... | Line 2952... | ||
2782 | size = map_and_fenceable ? fence_size : obj->base.size; |
2952 | size = map_and_fenceable ? fence_size : obj->base.size; |
Line 2783... | Line 2953... | ||
2783 | 2953 | ||
2784 | /* If the object is bigger than the entire aperture, reject it early |
2954 | /* If the object is bigger than the entire aperture, reject it early |
2785 | * before evicting everything in a vain attempt to find space. |
2955 | * before evicting everything in a vain attempt to find space. |
2786 | */ |
2956 | */ |
2787 | if (obj->base.size > |
2957 | if (obj->base.size > gtt_max) { |
- | 2958 | DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n", |
|
2788 | (map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) { |
2959 | obj->base.size, |
2789 | DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
2960 | map_and_fenceable ? "mappable" : "total", |
2790 | FAIL(); |
2961 | gtt_max); |
2791 | return -E2BIG; |
2962 | return -E2BIG; |
Line 2792... | Line 2963... | ||
2792 | } |
2963 | } |
2793 | 2964 | ||
2794 | ret = i915_gem_object_get_pages(obj); |
2965 | ret = i915_gem_object_get_pages(obj); |
Line 2795... | Line 2966... | ||
2795 | if (ret) |
2966 | if (ret) |
Line -... | Line 2967... | ||
- | 2967 | return ret; |
|
- | 2968 | ||
2796 | return ret; |
2969 | i915_gem_object_pin_pages(obj); |
2797 | 2970 | ||
2798 | i915_gem_object_pin_pages(obj); |
2971 | BUG_ON(!i915_is_ggtt(vm)); |
2799 | 2972 | ||
2800 | node = kzalloc(sizeof(*node), GFP_KERNEL); |
2973 | vma = i915_gem_obj_lookup_or_create_vma(obj, vm); |
Line -... | Line 2974... | ||
- | 2974 | if (IS_ERR(vma)) { |
|
- | 2975 | ret = PTR_ERR(vma); |
|
- | 2976 | goto err_unpin; |
|
2801 | if (node == NULL) { |
2977 | } |
2802 | i915_gem_object_unpin_pages(obj); |
- | |
2803 | return -ENOMEM; |
2978 | |
2804 | } |
2979 | /* For now we only ever use 1 vma per object */ |
2805 | 2980 | WARN_ON(!list_is_singular(&obj->vma_list)); |
|
2806 | search_free: |
- | |
2807 | if (map_and_fenceable) |
- | |
2808 | ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node, |
2981 | |
2809 | size, alignment, obj->cache_level, |
2982 | search_free: |
Line 2810... | Line -... | ||
2810 | 0, dev_priv->gtt.mappable_end); |
- | |
2811 | else |
- | |
2812 | ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node, |
2983 | ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, |
2813 | size, alignment, obj->cache_level); |
2984 | size, alignment, |
2814 | if (ret) { |
2985 | obj->cache_level, 0, gtt_max, |
2815 | 2986 | DRM_MM_SEARCH_DEFAULT); |
|
2816 | i915_gem_object_unpin_pages(obj); |
2987 | if (ret) { |
2817 | kfree(node); |
2988 | |
2818 | return ret; |
2989 | goto err_free_vma; |
Line 2819... | Line 2990... | ||
2819 | } |
2990 | } |
2820 | if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) { |
2991 | if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node, |
2821 | i915_gem_object_unpin_pages(obj); |
- | |
2822 | drm_mm_put_block(node); |
2992 | obj->cache_level))) { |
2823 | return -EINVAL; |
- | |
2824 | } |
- | |
Line 2825... | Line 2993... | ||
2825 | 2993 | ret = -EINVAL; |
|
2826 | ret = i915_gem_gtt_prepare_object(obj); |
2994 | goto err_remove_node; |
Line 2827... | Line 2995... | ||
2827 | if (ret) { |
2995 | } |
2828 | i915_gem_object_unpin_pages(obj); |
2996 | |
Line 2829... | Line -... | ||
2829 | drm_mm_put_block(node); |
- | |
2830 | return ret; |
2997 | ret = i915_gem_gtt_prepare_object(obj); |
2831 | } |
2998 | if (ret) |
Line 2832... | Line 2999... | ||
2832 | 2999 | goto err_remove_node; |
|
2833 | list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list); |
3000 | |
Line 2834... | Line 3001... | ||
2834 | list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
3001 | list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); |
- | 3002 | list_add_tail(&vma->mm_list, &vm->inactive_list); |
|
Line 2835... | Line 3003... | ||
2835 | 3003 | ||
- | 3004 | if (i915_is_ggtt(vm)) { |
|
2836 | obj->gtt_space = node; |
3005 | bool mappable, fenceable; |
2837 | obj->gtt_offset = node->start; |
3006 | |
2838 | 3007 | fenceable = (vma->node.size == fence_size && |
|
- | 3008 | (vma->node.start & (fence_alignment - 1)) == 0); |
|
- | 3009 | ||
- | 3010 | mappable = (vma->node.start + obj->base.size <= |
|
- | 3011 | dev_priv->gtt.mappable_end); |
|
- | 3012 | ||
- | 3013 | obj->map_and_fenceable = mappable && fenceable; |
|
- | 3014 | } |
|
- | 3015 | ||
2839 | fenceable = |
3016 | WARN_ON(map_and_fenceable && !obj->map_and_fenceable); |
Line 2840... | Line 3017... | ||
2840 | node->size == fence_size && |
3017 | |
2841 | (node->start & (fence_alignment - 1)) == 0; |
3018 | trace_i915_vma_bind(vma, map_and_fenceable); |
- | 3019 | i915_gem_verify_gtt(dev); |
|
2842 | 3020 | return 0; |
|
2843 | mappable = |
3021 | |
2844 | obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end; |
3022 | err_remove_node: |
2845 | 3023 | drm_mm_remove_node(&vma->node); |
|
2846 | obj->map_and_fenceable = mappable && fenceable; |
3024 | err_free_vma: |
2847 | 3025 | i915_gem_vma_destroy(vma); |
|
2848 | i915_gem_object_unpin_pages(obj); |
3026 | err_unpin: |
Line 2849... | Line 3027... | ||
2849 | trace_i915_gem_object_bind(obj, map_and_fenceable); |
3027 | i915_gem_object_unpin_pages(obj); |
2850 | i915_gem_verify_gtt(dev); |
3028 | return ret; |
2851 | return 0; |
3029 | } |
2852 | } |
3030 | |
2853 | 3031 | bool |
|
2854 | void |
3032 | i915_gem_clflush_object(struct drm_i915_gem_object *obj, |
Line 2855... | Line 3033... | ||
2855 | i915_gem_clflush_object(struct drm_i915_gem_object *obj) |
3033 | bool force) |
2856 | { |
3034 | { |
2857 | /* If we don't have a page list set up, then we're not pinned |
3035 | /* If we don't have a page list set up, then we're not pinned |
2858 | * to GPU, and we can ignore the cache flush because it'll happen |
3036 | * to GPU, and we can ignore the cache flush because it'll happen |
2859 | * again at bind time. |
3037 | * again at bind time. |
2860 | */ |
3038 | */ |
2861 | if (obj->pages == NULL) |
3039 | if (obj->pages == NULL) |
2862 | return; |
3040 | return false; |
2863 | 3041 | ||
2864 | /* |
3042 | /* |
2865 | * Stolen memory is always coherent with the GPU as it is explicitly |
3043 | * Stolen memory is always coherent with the GPU as it is explicitly |
2866 | * marked as wc by the system, or the system is cache-coherent. |
3044 | * marked as wc by the system, or the system is cache-coherent. |
2867 | */ |
3045 | */ |
2868 | if (obj->stolen) |
3046 | if (obj->stolen) |
2869 | return; |
3047 | return false; |
Line 2925... | Line 3103... | ||
2925 | "mfence"); |
3103 | "mfence"); |
2926 | } |
3104 | } |
2927 | } |
3105 | } |
2928 | #endif |
3106 | #endif |
Line -... | Line 3107... | ||
- | 3107 | ||
2929 | 3108 | return true; |
|
Line 2930... | Line 3109... | ||
2930 | } |
3109 | } |
2931 | 3110 | ||
2932 | /** Flushes the GTT write domain for the object if it's dirty. */ |
3111 | /** Flushes the GTT write domain for the object if it's dirty. */ |
Line 2956... | Line 3135... | ||
2956 | old_write_domain); |
3135 | old_write_domain); |
2957 | } |
3136 | } |
Line 2958... | Line 3137... | ||
2958 | 3137 | ||
2959 | /** Flushes the CPU write domain for the object if it's dirty. */ |
3138 | /** Flushes the CPU write domain for the object if it's dirty. */ |
2960 | static void |
3139 | static void |
- | 3140 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj, |
|
2961 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) |
3141 | bool force) |
2962 | { |
3142 | { |
Line 2963... | Line 3143... | ||
2963 | uint32_t old_write_domain; |
3143 | uint32_t old_write_domain; |
2964 | 3144 | ||
Line 2965... | Line 3145... | ||
2965 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
3145 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
2966 | return; |
3146 | return; |
- | 3147 | ||
2967 | 3148 | if (i915_gem_clflush_object(obj, force)) |
|
2968 | i915_gem_clflush_object(obj); |
3149 | i915_gem_chipset_flush(obj->base.dev); |
Line 2969... | Line 3150... | ||
2969 | i915_gem_chipset_flush(obj->base.dev); |
3150 | |
2970 | old_write_domain = obj->base.write_domain; |
3151 | old_write_domain = obj->base.write_domain; |
Line 2987... | Line 3168... | ||
2987 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
3168 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
2988 | uint32_t old_write_domain, old_read_domains; |
3169 | uint32_t old_write_domain, old_read_domains; |
2989 | int ret; |
3170 | int ret; |
Line 2990... | Line 3171... | ||
2990 | 3171 | ||
2991 | /* Not valid to be called on unbound objects. */ |
3172 | /* Not valid to be called on unbound objects. */ |
2992 | if (obj->gtt_space == NULL) |
3173 | if (!i915_gem_obj_bound_any(obj)) |
Line 2993... | Line 3174... | ||
2993 | return -EINVAL; |
3174 | return -EINVAL; |
2994 | 3175 | ||
Line 2995... | Line 3176... | ||
2995 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
3176 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
2996 | return 0; |
3177 | return 0; |
2997 | 3178 | ||
Line 2998... | Line 3179... | ||
2998 | ret = i915_gem_object_wait_rendering(obj, !write); |
3179 | ret = i915_gem_object_wait_rendering(obj, !write); |
Line 2999... | Line 3180... | ||
2999 | if (ret) |
3180 | if (ret) |
3000 | return ret; |
3181 | return ret; |
3001 | 3182 | ||
3002 | i915_gem_object_flush_cpu_write_domain(obj); |
3183 | i915_gem_object_flush_cpu_write_domain(obj, false); |
Line 3025... | Line 3206... | ||
3025 | trace_i915_gem_object_change_domain(obj, |
3206 | trace_i915_gem_object_change_domain(obj, |
3026 | old_read_domains, |
3207 | old_read_domains, |
3027 | old_write_domain); |
3208 | old_write_domain); |
Line 3028... | Line 3209... | ||
3028 | 3209 | ||
3029 | /* And bump the LRU for this access */ |
3210 | /* And bump the LRU for this access */ |
- | 3211 | if (i915_gem_object_is_inactive(obj)) { |
|
- | 3212 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, |
|
- | 3213 | &dev_priv->gtt.base); |
|
- | 3214 | if (vma) |
|
3030 | if (i915_gem_object_is_inactive(obj)) |
3215 | list_move_tail(&vma->mm_list, |
- | 3216 | &dev_priv->gtt.base.inactive_list); |
|
- | 3217 | ||
Line 3031... | Line 3218... | ||
3031 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
3218 | } |
3032 | 3219 | ||
Line 3033... | Line 3220... | ||
3033 | return 0; |
3220 | return 0; |
3034 | } |
3221 | } |
3035 | 3222 | ||
3036 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
3223 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
3037 | enum i915_cache_level cache_level) |
3224 | enum i915_cache_level cache_level) |
- | 3225 | { |
|
3038 | { |
3226 | struct drm_device *dev = obj->base.dev; |
Line 3039... | Line 3227... | ||
3039 | struct drm_device *dev = obj->base.dev; |
3227 | drm_i915_private_t *dev_priv = dev->dev_private; |
3040 | drm_i915_private_t *dev_priv = dev->dev_private; |
3228 | struct i915_vma *vma; |
Line 3041... | Line 3229... | ||
3041 | int ret; |
3229 | int ret; |
3042 | 3230 | ||
3043 | if (obj->cache_level == cache_level) |
3231 | if (obj->cache_level == cache_level) |
3044 | return 0; |
3232 | return 0; |
Line -... | Line 3233... | ||
- | 3233 | ||
3045 | 3234 | if (obj->pin_count) { |
|
3046 | if (obj->pin_count) { |
3235 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
3047 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
3236 | return -EBUSY; |
3048 | return -EBUSY; |
3237 | } |
- | 3238 | ||
- | 3239 | list_for_each_entry(vma, &obj->vma_list, vma_link) { |
|
- | 3240 | if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) { |
|
3049 | } |
3241 | ret = i915_vma_unbind(vma); |
Line 3050... | Line 3242... | ||
3050 | 3242 | if (ret) |
|
3051 | if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { |
3243 | return ret; |
3052 | ret = i915_gem_object_unbind(obj); |
3244 | |
3053 | if (ret) |
3245 | break; |
Line 3054... | Line 3246... | ||
3054 | return ret; |
3246 | } |
Line 3074... | Line 3266... | ||
3074 | if (obj->has_global_gtt_mapping) |
3266 | if (obj->has_global_gtt_mapping) |
3075 | i915_gem_gtt_bind_object(obj, cache_level); |
3267 | i915_gem_gtt_bind_object(obj, cache_level); |
3076 | if (obj->has_aliasing_ppgtt_mapping) |
3268 | if (obj->has_aliasing_ppgtt_mapping) |
3077 | i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, |
3269 | i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, |
3078 | obj, cache_level); |
3270 | obj, cache_level); |
3079 | - | ||
3080 | obj->gtt_space->color = cache_level; |
- | |
3081 | } |
3271 | } |
Line -... | Line 3272... | ||
- | 3272 | ||
- | 3273 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
|
3082 | 3274 | vma->node.color = cache_level; |
|
- | 3275 | obj->cache_level = cache_level; |
|
- | 3276 | ||
3083 | if (cache_level == I915_CACHE_NONE) { |
3277 | if (cpu_write_needs_clflush(obj)) { |
Line 3084... | Line 3278... | ||
3084 | u32 old_read_domains, old_write_domain; |
3278 | u32 old_read_domains, old_write_domain; |
3085 | 3279 | ||
3086 | /* If we're coming from LLC cached, then we haven't |
3280 | /* If we're coming from LLC cached, then we haven't |
3087 | * actually been tracking whether the data is in the |
3281 | * actually been tracking whether the data is in the |
3088 | * CPU cache or not, since we only allow one bit set |
3282 | * CPU cache or not, since we only allow one bit set |
3089 | * in obj->write_domain and have been skipping the clflushes. |
3283 | * in obj->write_domain and have been skipping the clflushes. |
3090 | * Just set it to the CPU cache for now. |
3284 | * Just set it to the CPU cache for now. |
3091 | */ |
- | |
Line 3092... | Line 3285... | ||
3092 | WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); |
3285 | */ |
3093 | WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU); |
3286 | WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); |
Line 3094... | Line 3287... | ||
3094 | 3287 | ||
Line 3101... | Line 3294... | ||
3101 | trace_i915_gem_object_change_domain(obj, |
3294 | trace_i915_gem_object_change_domain(obj, |
3102 | old_read_domains, |
3295 | old_read_domains, |
3103 | old_write_domain); |
3296 | old_write_domain); |
3104 | } |
3297 | } |
Line 3105... | Line -... | ||
3105 | - | ||
3106 | obj->cache_level = cache_level; |
3298 | |
3107 | i915_gem_verify_gtt(dev); |
3299 | i915_gem_verify_gtt(dev); |
3108 | return 0; |
3300 | return 0; |
Line 3109... | Line 3301... | ||
3109 | } |
3301 | } |
Line 3129... | Line 3321... | ||
3129 | if (&obj->base == NULL) { |
3321 | if (&obj->base == NULL) { |
3130 | ret = -ENOENT; |
3322 | ret = -ENOENT; |
3131 | goto unlock; |
3323 | goto unlock; |
3132 | } |
3324 | } |
Line -... | Line 3325... | ||
- | 3325 | ||
- | 3326 | switch (obj->cache_level) { |
|
- | 3327 | case I915_CACHE_LLC: |
|
- | 3328 | case I915_CACHE_L3_LLC: |
|
- | 3329 | args->caching = I915_CACHING_CACHED; |
|
- | 3330 | break; |
|
- | 3331 | ||
- | 3332 | case I915_CACHE_WT: |
|
- | 3333 | args->caching = I915_CACHING_DISPLAY; |
|
- | 3334 | break; |
|
- | 3335 | ||
3133 | 3336 | default: |
|
- | 3337 | args->caching = I915_CACHING_NONE; |
|
- | 3338 | break; |
|
Line 3134... | Line 3339... | ||
3134 | args->caching = obj->cache_level != I915_CACHE_NONE; |
3339 | } |
3135 | 3340 | ||
3136 | drm_gem_object_unreference(&obj->base); |
3341 | drm_gem_object_unreference(&obj->base); |
3137 | unlock: |
3342 | unlock: |
Line 3158... | Line 3363... | ||
3158 | level = I915_CACHE_NONE; |
3363 | level = I915_CACHE_NONE; |
3159 | break; |
3364 | break; |
3160 | case I915_CACHING_CACHED: |
3365 | case I915_CACHING_CACHED: |
3161 | level = I915_CACHE_LLC; |
3366 | level = I915_CACHE_LLC; |
3162 | break; |
3367 | break; |
- | 3368 | case I915_CACHING_DISPLAY: |
|
- | 3369 | level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE; |
|
- | 3370 | break; |
|
3163 | default: |
3371 | default: |
3164 | return -EINVAL; |
3372 | return -EINVAL; |
3165 | } |
3373 | } |
Line 3166... | Line 3374... | ||
3166 | 3374 | ||
Line 3180... | Line 3388... | ||
3180 | unlock: |
3388 | unlock: |
3181 | mutex_unlock(&dev->struct_mutex); |
3389 | mutex_unlock(&dev->struct_mutex); |
3182 | return ret; |
3390 | return ret; |
3183 | } |
3391 | } |
Line -... | Line 3392... | ||
- | 3392 | ||
- | 3393 | static bool is_pin_display(struct drm_i915_gem_object *obj) |
|
- | 3394 | { |
|
- | 3395 | /* There are 3 sources that pin objects: |
|
- | 3396 | * 1. The display engine (scanouts, sprites, cursors); |
|
- | 3397 | * 2. Reservations for execbuffer; |
|
- | 3398 | * 3. The user. |
|
- | 3399 | * |
|
- | 3400 | * We can ignore reservations as we hold the struct_mutex and |
|
- | 3401 | * are only called outside of the reservation path. The user |
|
- | 3402 | * can only increment pin_count once, and so if after |
|
- | 3403 | * subtracting the potential reference by the user, any pin_count |
|
- | 3404 | * remains, it must be due to another use by the display engine. |
|
- | 3405 | */ |
|
- | 3406 | return obj->pin_count - !!obj->user_pin_count; |
|
- | 3407 | } |
|
3184 | 3408 | ||
3185 | /* |
3409 | /* |
3186 | * Prepare buffer for display plane (scanout, cursors, etc). |
3410 | * Prepare buffer for display plane (scanout, cursors, etc). |
3187 | * Can be called from an uninterruptible phase (modesetting) and allows |
3411 | * Can be called from an uninterruptible phase (modesetting) and allows |
3188 | * any flushes to be pipelined (for pageflips). |
3412 | * any flushes to be pipelined (for pageflips). |
Line 3199... | Line 3423... | ||
3199 | ret = i915_gem_object_sync(obj, pipelined); |
3423 | ret = i915_gem_object_sync(obj, pipelined); |
3200 | if (ret) |
3424 | if (ret) |
3201 | return ret; |
3425 | return ret; |
3202 | } |
3426 | } |
Line -... | Line 3427... | ||
- | 3427 | ||
- | 3428 | /* Mark the pin_display early so that we account for the |
|
- | 3429 | * display coherency whilst setting up the cache domains. |
|
- | 3430 | */ |
|
- | 3431 | obj->pin_display = true; |
|
3203 | 3432 | ||
3204 | /* The display engine is not coherent with the LLC cache on gen6. As |
3433 | /* The display engine is not coherent with the LLC cache on gen6. As |
3205 | * a result, we make sure that the pinning that is about to occur is |
3434 | * a result, we make sure that the pinning that is about to occur is |
3206 | * done with uncached PTEs. This is lowest common denominator for all |
3435 | * done with uncached PTEs. This is lowest common denominator for all |
3207 | * chipsets. |
3436 | * chipsets. |
3208 | * |
3437 | * |
3209 | * However for gen6+, we could do better by using the GFDT bit instead |
3438 | * However for gen6+, we could do better by using the GFDT bit instead |
3210 | * of uncaching, which would allow us to flush all the LLC-cached data |
3439 | * of uncaching, which would allow us to flush all the LLC-cached data |
3211 | * with that bit in the PTE to main memory with just one PIPE_CONTROL. |
3440 | * with that bit in the PTE to main memory with just one PIPE_CONTROL. |
3212 | */ |
3441 | */ |
- | 3442 | ret = i915_gem_object_set_cache_level(obj, |
|
3213 | ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); |
3443 | HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE); |
3214 | if (ret) |
3444 | if (ret) |
Line 3215... | Line 3445... | ||
3215 | return ret; |
3445 | goto err_unpin_display; |
3216 | 3446 | ||
3217 | /* As the user may map the buffer once pinned in the display plane |
3447 | /* As the user may map the buffer once pinned in the display plane |
3218 | * (e.g. libkms for the bootup splash), we have to ensure that we |
3448 | * (e.g. libkms for the bootup splash), we have to ensure that we |
3219 | * always use map_and_fenceable for all scanout buffers. |
3449 | * always use map_and_fenceable for all scanout buffers. |
3220 | */ |
3450 | */ |
3221 | ret = i915_gem_object_pin(obj, alignment, true, false); |
3451 | ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false); |
Line 3222... | Line 3452... | ||
3222 | if (ret) |
3452 | if (ret) |
Line 3223... | Line 3453... | ||
3223 | return ret; |
3453 | goto err_unpin_display; |
3224 | 3454 | ||
Line 3225... | Line 3455... | ||
3225 | i915_gem_object_flush_cpu_write_domain(obj); |
3455 | i915_gem_object_flush_cpu_write_domain(obj, true); |
Line 3236... | Line 3466... | ||
3236 | trace_i915_gem_object_change_domain(obj, |
3466 | trace_i915_gem_object_change_domain(obj, |
3237 | old_read_domains, |
3467 | old_read_domains, |
3238 | old_write_domain); |
3468 | old_write_domain); |
Line 3239... | Line 3469... | ||
3239 | 3469 | ||
- | 3470 | return 0; |
|
- | 3471 | ||
- | 3472 | err_unpin_display: |
|
- | 3473 | obj->pin_display = is_pin_display(obj); |
|
- | 3474 | return ret; |
|
- | 3475 | } |
|
- | 3476 | ||
- | 3477 | void |
|
- | 3478 | i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) |
|
- | 3479 | { |
|
- | 3480 | i915_gem_object_unpin(obj); |
|
3240 | return 0; |
3481 | obj->pin_display = is_pin_display(obj); |
Line 3241... | Line 3482... | ||
3241 | } |
3482 | } |
3242 | 3483 | ||
3243 | int |
3484 | int |
Line 3281... | Line 3522... | ||
3281 | old_write_domain = obj->base.write_domain; |
3522 | old_write_domain = obj->base.write_domain; |
3282 | old_read_domains = obj->base.read_domains; |
3523 | old_read_domains = obj->base.read_domains; |
Line 3283... | Line 3524... | ||
3283 | 3524 | ||
3284 | /* Flush the CPU cache if it's still invalid. */ |
3525 | /* Flush the CPU cache if it's still invalid. */ |
3285 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { |
3526 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { |
Line 3286... | Line 3527... | ||
3286 | i915_gem_clflush_object(obj); |
3527 | i915_gem_clflush_object(obj, false); |
3287 | 3528 | ||
Line 3288... | Line 3529... | ||
3288 | obj->base.read_domains |= I915_GEM_DOMAIN_CPU; |
3529 | obj->base.read_domains |= I915_GEM_DOMAIN_CPU; |
Line 3359... | Line 3600... | ||
3359 | return ret; |
3600 | return ret; |
3360 | } |
3601 | } |
Line 3361... | Line 3602... | ||
3361 | 3602 | ||
3362 | int |
3603 | int |
- | 3604 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
|
3363 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
3605 | struct i915_address_space *vm, |
3364 | uint32_t alignment, |
3606 | uint32_t alignment, |
3365 | bool map_and_fenceable, |
3607 | bool map_and_fenceable, |
3366 | bool nonblocking) |
3608 | bool nonblocking) |
- | 3609 | { |
|
3367 | { |
3610 | struct i915_vma *vma; |
Line 3368... | Line 3611... | ||
3368 | int ret; |
3611 | int ret; |
3369 | 3612 | ||
Line -... | Line 3613... | ||
- | 3613 | if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) |
|
- | 3614 | return -EBUSY; |
|
- | 3615 | ||
- | 3616 | WARN_ON(map_and_fenceable && !i915_is_ggtt(vm)); |
|
- | 3617 | ||
3370 | if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) |
3618 | vma = i915_gem_obj_to_vma(obj, vm); |
3371 | return -EBUSY; |
3619 | |
3372 | 3620 | if (vma) { |
|
3373 | if (obj->gtt_space != NULL) { |
3621 | if ((alignment && |
3374 | if ((alignment && obj->gtt_offset & (alignment - 1)) || |
3622 | vma->node.start & (alignment - 1)) || |
3375 | (map_and_fenceable && !obj->map_and_fenceable)) { |
3623 | (map_and_fenceable && !obj->map_and_fenceable)) { |
3376 | WARN(obj->pin_count, |
3624 | WARN(obj->pin_count, |
3377 | "bo is already pinned with incorrect alignment:" |
3625 | "bo is already pinned with incorrect alignment:" |
3378 | " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," |
3626 | " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d," |
3379 | " obj->map_and_fenceable=%d\n", |
3627 | " obj->map_and_fenceable=%d\n", |
3380 | obj->gtt_offset, alignment, |
3628 | i915_gem_obj_offset(obj, vm), alignment, |
3381 | map_and_fenceable, |
3629 | map_and_fenceable, |
3382 | obj->map_and_fenceable); |
3630 | obj->map_and_fenceable); |
3383 | ret = i915_gem_object_unbind(obj); |
3631 | ret = i915_vma_unbind(vma); |
3384 | if (ret) |
3632 | if (ret) |
Line 3385... | Line 3633... | ||
3385 | return ret; |
3633 | return ret; |
3386 | } |
3634 | } |
Line 3387... | Line 3635... | ||
3387 | } |
3635 | } |
3388 | 3636 | ||
3389 | if (obj->gtt_space == NULL) { |
3637 | if (!i915_gem_obj_bound(obj, vm)) { |
3390 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
3638 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
3391 | 3639 | ||
Line 3410... | Line 3658... | ||
3410 | 3658 | ||
3411 | void |
3659 | void |
3412 | i915_gem_object_unpin(struct drm_i915_gem_object *obj) |
3660 | i915_gem_object_unpin(struct drm_i915_gem_object *obj) |
3413 | { |
3661 | { |
3414 | BUG_ON(obj->pin_count == 0); |
3662 | BUG_ON(obj->pin_count == 0); |
Line 3415... | Line 3663... | ||
3415 | BUG_ON(obj->gtt_space == NULL); |
3663 | BUG_ON(!i915_gem_obj_bound_any(obj)); |
3416 | 3664 | ||
3417 | if (--obj->pin_count == 0) |
3665 | if (--obj->pin_count == 0) |
Line 3454... | Line 3702... | ||
3454 | ret = -EINVAL; |
3702 | ret = -EINVAL; |
3455 | goto out; |
3703 | goto out; |
3456 | } |
3704 | } |
Line 3457... | Line 3705... | ||
3457 | 3705 | ||
3458 | if (obj->user_pin_count == 0) { |
3706 | if (obj->user_pin_count == 0) { |
3459 | ret = i915_gem_object_pin(obj, args->alignment, true, false); |
3707 | ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false); |
3460 | if (ret) |
3708 | if (ret) |
3461 | goto out; |
3709 | goto out; |
Line 3462... | Line 3710... | ||
3462 | } |
3710 | } |
3463 | 3711 | ||
Line 3464... | Line -... | ||
3464 | obj->user_pin_count++; |
- | |
3465 | obj->pin_filp = file; |
- | |
3466 | - | ||
3467 | /* XXX - flush the CPU caches for pinned objects |
- | |
3468 | * as the X server doesn't manage domains yet |
3712 | obj->user_pin_count++; |
3469 | */ |
3713 | obj->pin_filp = file; |
3470 | i915_gem_object_flush_cpu_write_domain(obj); |
3714 | |
3471 | args->offset = obj->gtt_offset; |
3715 | args->offset = i915_gem_obj_ggtt_offset(obj); |
3472 | out: |
3716 | out: |
3473 | drm_gem_object_unreference(&obj->base); |
3717 | drm_gem_object_unreference(&obj->base); |
Line 3618... | Line 3862... | ||
3618 | #endif |
3862 | #endif |
Line 3619... | Line 3863... | ||
3619 | 3863 | ||
3620 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
3864 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
3621 | const struct drm_i915_gem_object_ops *ops) |
3865 | const struct drm_i915_gem_object_ops *ops) |
3622 | { |
- | |
3623 | INIT_LIST_HEAD(&obj->mm_list); |
3866 | { |
3624 | INIT_LIST_HEAD(&obj->gtt_list); |
3867 | INIT_LIST_HEAD(&obj->global_list); |
3625 | INIT_LIST_HEAD(&obj->ring_list); |
3868 | INIT_LIST_HEAD(&obj->ring_list); |
- | 3869 | INIT_LIST_HEAD(&obj->exec_list); |
|
- | 3870 | INIT_LIST_HEAD(&obj->obj_exec_link); |
|
Line 3626... | Line 3871... | ||
3626 | INIT_LIST_HEAD(&obj->exec_list); |
3871 | INIT_LIST_HEAD(&obj->vma_list); |
Line 3627... | Line 3872... | ||
3627 | 3872 | ||
3628 | obj->ops = ops; |
3873 | obj->ops = ops; |
Line 3646... | Line 3891... | ||
3646 | struct drm_i915_gem_object *obj; |
3891 | struct drm_i915_gem_object *obj; |
3647 | struct address_space *mapping; |
3892 | struct address_space *mapping; |
3648 | gfp_t mask; |
3893 | gfp_t mask; |
Line 3649... | Line 3894... | ||
3649 | 3894 | ||
3650 | obj = i915_gem_object_alloc(dev); |
- | |
3651 | - | ||
3652 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); |
3895 | obj = i915_gem_object_alloc(dev); |
3653 | if (obj == NULL) |
3896 | if (obj == NULL) |
3654 | { |
3897 | { |
3655 | FAIL(); |
3898 | FAIL(); |
3656 | return NULL; |
3899 | return NULL; |
Line 3657... | Line 3900... | ||
3657 | }; |
3900 | }; |
3658 | 3901 | ||
3659 | if (drm_gem_object_init(dev, &obj->base, size) != 0) { |
- | |
3660 | kfree(obj); |
3902 | if (drm_gem_object_init(dev, &obj->base, size) != 0) { |
3661 | FAIL(); |
3903 | i915_gem_object_free(obj); |
Line 3662... | Line 3904... | ||
3662 | return NULL; |
3904 | return NULL; |
Line 3698... | Line 3940... | ||
3698 | void i915_gem_free_object(struct drm_gem_object *gem_obj) |
3940 | void i915_gem_free_object(struct drm_gem_object *gem_obj) |
3699 | { |
3941 | { |
3700 | struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); |
3942 | struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); |
3701 | struct drm_device *dev = obj->base.dev; |
3943 | struct drm_device *dev = obj->base.dev; |
3702 | drm_i915_private_t *dev_priv = dev->dev_private; |
3944 | drm_i915_private_t *dev_priv = dev->dev_private; |
- | 3945 | struct i915_vma *vma, *next; |
|
Line 3703... | Line 3946... | ||
3703 | 3946 | ||
Line 3704... | Line -... | ||
3704 | trace_i915_gem_object_destroy(obj); |
- | |
3705 | - | ||
3706 | // if (obj->phys_obj) |
- | |
Line 3707... | Line 3947... | ||
3707 | // i915_gem_detach_phys_object(dev, obj); |
3947 | trace_i915_gem_object_destroy(obj); |
- | 3948 | ||
- | 3949 | ||
- | 3950 | obj->pin_count = 0; |
|
- | 3951 | /* NB: 0 or 1 elements */ |
|
- | 3952 | WARN_ON(!list_empty(&obj->vma_list) && |
|
3708 | // printf("%s obj %p\n", __FUNCTION__, obj); |
3953 | !list_is_singular(&obj->vma_list)); |
3709 | 3954 | list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { |
|
Line 3710... | Line 3955... | ||
3710 | obj->pin_count = 0; |
3955 | int ret = i915_vma_unbind(vma); |
3711 | if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { |
3956 | if (WARN_ON(ret == -ERESTARTSYS)) { |
Line 3712... | Line 3957... | ||
3712 | bool was_interruptible; |
3957 | bool was_interruptible; |
Line 3713... | Line 3958... | ||
3713 | 3958 | ||
3714 | was_interruptible = dev_priv->mm.interruptible; |
3959 | was_interruptible = dev_priv->mm.interruptible; |
- | 3960 | dev_priv->mm.interruptible = false; |
|
- | 3961 | ||
- | 3962 | WARN_ON(i915_vma_unbind(vma)); |
|
- | 3963 | ||
- | 3964 | dev_priv->mm.interruptible = was_interruptible; |
|
- | 3965 | } |
|
Line -... | Line 3966... | ||
- | 3966 | } |
|
3715 | dev_priv->mm.interruptible = false; |
3967 | |
3716 | 3968 | /* Stolen objects don't hold a ref, but do hold pin count. Fix that up |
|
3717 | WARN_ON(i915_gem_object_unbind(obj)); |
3969 | * before progressing. */ |
- | 3970 | if (obj->stolen) |
|
Line 3718... | Line 3971... | ||
3718 | 3971 | i915_gem_object_unpin_pages(obj); |
|
Line 3719... | Line -... | ||
3719 | dev_priv->mm.interruptible = was_interruptible; |
- | |
3720 | } |
- | |
Line 3721... | Line 3972... | ||
3721 | 3972 | ||
3722 | obj->pages_pin_count = 0; |
3973 | if (WARN_ON(obj->pages_pin_count)) |
3723 | i915_gem_object_put_pages(obj); |
3974 | obj->pages_pin_count = 0; |
3724 | // i915_gem_object_free_mmap_offset(obj); |
3975 | i915_gem_object_put_pages(obj); |
Line 3736... | Line 3987... | ||
3736 | 3987 | ||
3737 | drm_gem_object_release(&obj->base); |
3988 | drm_gem_object_release(&obj->base); |
Line 3738... | Line 3989... | ||
3738 | i915_gem_info_remove_obj(dev_priv, obj->base.size); |
3989 | i915_gem_info_remove_obj(dev_priv, obj->base.size); |
- | 3990 | ||
- | 3991 | kfree(obj->bit_17); |
|
- | 3992 | i915_gem_object_free(obj); |
|
- | 3993 | } |
|
- | 3994 | ||
- | 3995 | struct i915_vma *i915_gem_vma_create(struct drm_i915_gem_object *obj, |
|
- | 3996 | struct i915_address_space *vm) |
|
- | 3997 | { |
|
- | 3998 | struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); |
|
- | 3999 | if (vma == NULL) |
|
- | 4000 | return ERR_PTR(-ENOMEM); |
|
- | 4001 | ||
- | 4002 | INIT_LIST_HEAD(&vma->vma_link); |
|
- | 4003 | INIT_LIST_HEAD(&vma->mm_list); |
|
- | 4004 | INIT_LIST_HEAD(&vma->exec_list); |
|
- | 4005 | vma->vm = vm; |
|
- | 4006 | vma->obj = obj; |
|
- | 4007 | ||
- | 4008 | /* Keep GGTT vmas first to make debug easier */ |
|
- | 4009 | if (i915_is_ggtt(vm)) |
|
- | 4010 | list_add(&vma->vma_link, &obj->vma_list); |
|
- | 4011 | else |
|
- | 4012 | list_add_tail(&vma->vma_link, &obj->vma_list); |
|
- | 4013 | ||
- | 4014 | return vma; |
|
- | 4015 | } |
|
- | 4016 | ||
- | 4017 | void i915_gem_vma_destroy(struct i915_vma *vma) |
|
- | 4018 | { |
|
3739 | 4019 | WARN_ON(vma->node.allocated); |
|
3740 | kfree(obj->bit_17); |
4020 | list_del(&vma->vma_link); |
Line 3741... | Line 4021... | ||
3741 | kfree(obj); |
4021 | kfree(vma); |
3742 | } |
4022 | } |
3743 | 4023 | ||
3744 | #if 0 |
4024 | #if 0 |
3745 | int |
4025 | int |
3746 | i915_gem_idle(struct drm_device *dev) |
4026 | i915_gem_idle(struct drm_device *dev) |
Line 3747... | Line -... | ||
3747 | { |
- | |
3748 | drm_i915_private_t *dev_priv = dev->dev_private; |
- | |
3749 | int ret; |
4027 | { |
3750 | 4028 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
3751 | mutex_lock(&dev->struct_mutex); |
4029 | int ret; |
3752 | 4030 | ||
Line 3753... | Line 4031... | ||
3753 | if (dev_priv->mm.suspended) { |
4031 | if (dev_priv->ums.mm_suspended) { |
Line 3764... | Line 4042... | ||
3764 | 4042 | ||
3765 | /* Under UMS, be paranoid and evict. */ |
4043 | /* Under UMS, be paranoid and evict. */ |
3766 | if (!drm_core_check_feature(dev, DRIVER_MODESET)) |
4044 | if (!drm_core_check_feature(dev, DRIVER_MODESET)) |
Line 3767... | Line -... | ||
3767 | i915_gem_evict_everything(dev); |
- | |
3768 | - | ||
3769 | /* Hack! Don't let anybody do execbuf while we don't control the chip. |
- | |
3770 | * We need to replace this with a semaphore, or something. |
- | |
3771 | * And not confound mm.suspended! |
- | |
3772 | */ |
4045 | i915_gem_evict_everything(dev); |
Line 3773... | Line 4046... | ||
3773 | dev_priv->mm.suspended = 1; |
4046 | |
3774 | del_timer_sync(&dev_priv->gpu_error.hangcheck_timer); |
4047 | del_timer_sync(&dev_priv->gpu_error.hangcheck_timer); |
Line 3775... | Line -... | ||
3775 | - | ||
3776 | i915_kernel_lost_context(dev); |
- | |
3777 | i915_gem_cleanup_ringbuffer(dev); |
4048 | |
3778 | 4049 | i915_kernel_lost_context(dev); |
|
Line 3779... | Line 4050... | ||
3779 | mutex_unlock(&dev->struct_mutex); |
4050 | i915_gem_cleanup_ringbuffer(dev); |
3780 | 4051 | ||
Line 3875... | Line 4146... | ||
3875 | ret = intel_init_blt_ring_buffer(dev); |
4146 | ret = intel_init_blt_ring_buffer(dev); |
3876 | if (ret) |
4147 | if (ret) |
3877 | goto cleanup_bsd_ring; |
4148 | goto cleanup_bsd_ring; |
3878 | } |
4149 | } |
Line -... | Line 4150... | ||
- | 4150 | ||
3879 | 4151 | if (HAS_VEBOX(dev)) { |
|
3880 | ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); |
4152 | ret = intel_init_vebox_ring_buffer(dev); |
3881 | if (ret) |
4153 | if (ret) |
- | 4154 | goto cleanup_blt_ring; |
|
- | 4155 | } |
|
- | 4156 | ||
- | 4157 | ||
- | 4158 | ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); |
|
- | 4159 | if (ret) |
|
Line 3882... | Line 4160... | ||
3882 | goto cleanup_blt_ring; |
4160 | goto cleanup_vebox_ring; |
Line -... | Line 4161... | ||
- | 4161 | ||
- | 4162 | return 0; |
|
3883 | 4163 | ||
3884 | return 0; |
4164 | cleanup_vebox_ring: |
3885 | 4165 | intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); |
|
3886 | cleanup_blt_ring: |
4166 | cleanup_blt_ring: |
3887 | intel_cleanup_ring_buffer(&dev_priv->ring[BCS]); |
4167 | intel_cleanup_ring_buffer(&dev_priv->ring[BCS]); |
Line 3900... | Line 4180... | ||
3900 | int ret; |
4180 | int ret; |
Line 3901... | Line 4181... | ||
3901 | 4181 | ||
3902 | if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) |
4182 | if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) |
Line 3903... | Line 4183... | ||
3903 | return -EIO; |
4183 | return -EIO; |
3904 | 4184 | ||
Line 3905... | Line 4185... | ||
3905 | if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) |
4185 | if (dev_priv->ellc_size) |
3906 | I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); |
4186 | I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); |
3907 | 4187 | ||
3908 | if (HAS_PCH_NOP(dev)) { |
4188 | if (HAS_PCH_NOP(dev)) { |
Line 3979... | Line 4259... | ||
3979 | 4259 | ||
3980 | int |
4260 | int |
3981 | i915_gem_entervt_ioctl(struct drm_device *dev, void *data, |
4261 | i915_gem_entervt_ioctl(struct drm_device *dev, void *data, |
3982 | struct drm_file *file_priv) |
4262 | struct drm_file *file_priv) |
3983 | { |
4263 | { |
3984 | drm_i915_private_t *dev_priv = dev->dev_private; |
4264 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 3985... | Line 4265... | ||
3985 | int ret; |
4265 | int ret; |
3986 | 4266 | ||
Line 3991... | Line 4271... | ||
3991 | DRM_ERROR("Reenabling wedged hardware, good luck\n"); |
4271 | DRM_ERROR("Reenabling wedged hardware, good luck\n"); |
3992 | atomic_set(&dev_priv->gpu_error.reset_counter, 0); |
4272 | atomic_set(&dev_priv->gpu_error.reset_counter, 0); |
3993 | } |
4273 | } |
Line 3994... | Line 4274... | ||
3994 | 4274 | ||
3995 | mutex_lock(&dev->struct_mutex); |
4275 | mutex_lock(&dev->struct_mutex); |
Line 3996... | Line 4276... | ||
3996 | dev_priv->mm.suspended = 0; |
4276 | dev_priv->ums.mm_suspended = 0; |
3997 | 4277 | ||
3998 | ret = i915_gem_init_hw(dev); |
4278 | ret = i915_gem_init_hw(dev); |
3999 | if (ret != 0) { |
4279 | if (ret != 0) { |
4000 | mutex_unlock(&dev->struct_mutex); |
4280 | mutex_unlock(&dev->struct_mutex); |
Line 4001... | Line 4281... | ||
4001 | return ret; |
4281 | return ret; |
4002 | } |
4282 | } |
Line 4003... | Line 4283... | ||
4003 | 4283 | ||
4004 | BUG_ON(!list_empty(&dev_priv->mm.active_list)); |
4284 | BUG_ON(!list_empty(&dev_priv->gtt.base.active_list)); |
4005 | mutex_unlock(&dev->struct_mutex); |
4285 | mutex_unlock(&dev->struct_mutex); |
Line 4011... | Line 4291... | ||
4011 | return 0; |
4291 | return 0; |
Line 4012... | Line 4292... | ||
4012 | 4292 | ||
4013 | cleanup_ringbuffer: |
4293 | cleanup_ringbuffer: |
4014 | mutex_lock(&dev->struct_mutex); |
4294 | mutex_lock(&dev->struct_mutex); |
4015 | i915_gem_cleanup_ringbuffer(dev); |
4295 | i915_gem_cleanup_ringbuffer(dev); |
4016 | dev_priv->mm.suspended = 1; |
4296 | dev_priv->ums.mm_suspended = 1; |
Line 4017... | Line 4297... | ||
4017 | mutex_unlock(&dev->struct_mutex); |
4297 | mutex_unlock(&dev->struct_mutex); |
4018 | 4298 | ||
Line 4019... | Line 4299... | ||
4019 | return ret; |
4299 | return ret; |
4020 | } |
4300 | } |
4021 | 4301 | ||
4022 | int |
4302 | int |
- | 4303 | i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, |
|
- | 4304 | struct drm_file *file_priv) |
|
- | 4305 | { |
|
4023 | i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, |
4306 | struct drm_i915_private *dev_priv = dev->dev_private; |
4024 | struct drm_file *file_priv) |
4307 | int ret; |
Line 4025... | Line 4308... | ||
4025 | { |
4308 | |
- | 4309 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
|
- | 4310 | return 0; |
|
4026 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
4311 | |
- | 4312 | drm_irq_uninstall(dev); |
|
- | 4313 | ||
- | 4314 | mutex_lock(&dev->struct_mutex); |
|
- | 4315 | ret = i915_gem_idle(dev); |
|
- | 4316 | ||
- | 4317 | /* Hack! Don't let anybody do execbuf while we don't control the chip. |
|
- | 4318 | * We need to replace this with a semaphore, or something. |
|
- | 4319 | * And not confound ums.mm_suspended! |
|
- | 4320 | */ |
|
- | 4321 | if (ret != 0) |
|
4027 | return 0; |
4322 | dev_priv->ums.mm_suspended = 1; |
Line 4028... | Line 4323... | ||
4028 | 4323 | mutex_unlock(&dev->struct_mutex); |
|
4029 | drm_irq_uninstall(dev); |
4324 | |
4030 | return i915_gem_idle(dev); |
4325 | return ret; |
4031 | } |
4326 | } |
Line 4032... | Line 4327... | ||
4032 | 4327 | ||
4033 | void |
4328 | void |
Line -... | Line 4329... | ||
- | 4329 | i915_gem_lastclose(struct drm_device *dev) |
|
4034 | i915_gem_lastclose(struct drm_device *dev) |
4330 | { |
4035 | { |
4331 | int ret; |
4036 | int ret; |
4332 | |
- | 4333 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
|
4037 | 4334 | return; |
|
4038 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
4335 | |
Line 4039... | Line 4336... | ||
4039 | return; |
4336 | mutex_lock(&dev->struct_mutex); |
4040 | 4337 | ret = i915_gem_idle(dev); |
|
4041 | ret = i915_gem_idle(dev); |
4338 | if (ret) |
4042 | if (ret) |
4339 | DRM_ERROR("failed to idle hardware: %d\n", ret); |
4043 | DRM_ERROR("failed to idle hardware: %d\n", ret); |
4340 | mutex_unlock(&dev->struct_mutex); |
4044 | } |
4341 | } |
Line -... | Line 4342... | ||
- | 4342 | #endif |
|
- | 4343 | ||
- | 4344 | static void |
|
- | 4345 | init_ring_lists(struct intel_ring_buffer *ring) |
|
- | 4346 | { |
|
- | 4347 | INIT_LIST_HEAD(&ring->active_list); |
|
- | 4348 | INIT_LIST_HEAD(&ring->request_list); |
|
- | 4349 | } |
|
- | 4350 | ||
- | 4351 | static void i915_init_vm(struct drm_i915_private *dev_priv, |
|
4045 | #endif |
4352 | struct i915_address_space *vm) |
4046 | 4353 | { |
|
4047 | static void |
4354 | vm->dev = dev_priv->dev; |
4048 | init_ring_lists(struct intel_ring_buffer *ring) |
4355 | INIT_LIST_HEAD(&vm->active_list); |
4049 | { |
4356 | INIT_LIST_HEAD(&vm->inactive_list); |
Line 4050... | Line 4357... | ||
4050 | INIT_LIST_HEAD(&ring->active_list); |
4357 | INIT_LIST_HEAD(&vm->global_link); |
4051 | INIT_LIST_HEAD(&ring->request_list); |
4358 | list_add(&vm->global_link, &dev_priv->vm_list); |
- | 4359 | } |
|
4052 | } |
4360 | |
4053 | 4361 | void |
|
4054 | void |
4362 | i915_gem_load(struct drm_device *dev) |
4055 | i915_gem_load(struct drm_device *dev) |
4363 | { |
4056 | { |
4364 | drm_i915_private_t *dev_priv = dev->dev_private; |
Line 4091... | Line 4399... | ||
4091 | 4399 | ||
Line 4092... | Line 4400... | ||
4092 | i915_gem_detect_bit_6_swizzle(dev); |
4400 | i915_gem_detect_bit_6_swizzle(dev); |
Line 4093... | Line -... | ||
4093 | - | ||
4094 | dev_priv->mm.interruptible = true; |
- | |
4095 | - | ||
4096 | // dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink; |
4401 | |
Line -... | Line 4402... | ||
- | 4402 | dev_priv->mm.interruptible = true; |
|
- | 4403 | ||
- | 4404 | } |
|
- | 4405 | ||
- | 4406 | #if 0 |
|
- | 4407 | /* |
|
- | 4408 | * Create a physically contiguous memory object for this object |
|
- | 4409 | * e.g. for cursor + overlay regs |
|
- | 4410 | */ |
|
- | 4411 | static int i915_gem_init_phys_object(struct drm_device *dev, |
|
- | 4412 | int id, int size, int align) |
|
- | 4413 | { |
|
- | 4414 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 4415 | struct drm_i915_gem_phys_object *phys_obj; |
|
- | 4416 | int ret; |
|
- | 4417 | ||
- | 4418 | if (dev_priv->mm.phys_objs[id - 1] || !size) |
|
- | 4419 | return 0; |
|
- | 4420 | ||
- | 4421 | phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL); |
|
- | 4422 | if (!phys_obj) |
|
- | 4423 | return -ENOMEM; |
|
- | 4424 | ||
- | 4425 | phys_obj->id = id; |
|
- | 4426 | ||
- | 4427 | phys_obj->handle = drm_pci_alloc(dev, size, align); |
|
- | 4428 | if (!phys_obj->handle) { |
|
- | 4429 | ret = -ENOMEM; |
|
- | 4430 | goto kfree_obj; |
|
- | 4431 | } |
|
- | 4432 | #ifdef CONFIG_X86 |
|
- | 4433 | set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); |
|
- | 4434 | #endif |
|
- | 4435 | ||
- | 4436 | dev_priv->mm.phys_objs[id - 1] = phys_obj; |
|
- | 4437 | ||
- | 4438 | return 0; |
|
- | 4439 | kfree_obj: |
|
- | 4440 | kfree(phys_obj); |
|
- | 4441 | return ret; |
|
- | 4442 | } |
|
- | 4443 | ||
- | 4444 | static void i915_gem_free_phys_object(struct drm_device *dev, int id) |
|
- | 4445 | { |
|
- | 4446 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 4447 | struct drm_i915_gem_phys_object *phys_obj; |
|
- | 4448 | ||
- | 4449 | if (!dev_priv->mm.phys_objs[id - 1]) |
|
- | 4450 | return; |
|
- | 4451 | ||
- | 4452 | phys_obj = dev_priv->mm.phys_objs[id - 1]; |
|
- | 4453 | if (phys_obj->cur_obj) { |
|
- | 4454 | i915_gem_detach_phys_object(dev, phys_obj->cur_obj); |
|
- | 4455 | } |
|
- | 4456 | ||
- | 4457 | #ifdef CONFIG_X86 |
|
- | 4458 | set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); |
|
- | 4459 | #endif |
|
- | 4460 | drm_pci_free(dev, phys_obj->handle); |
|
- | 4461 | kfree(phys_obj); |
|
- | 4462 | dev_priv->mm.phys_objs[id - 1] = NULL; |
|
- | 4463 | } |
|
- | 4464 | ||
- | 4465 | void i915_gem_free_all_phys_object(struct drm_device *dev) |
|
- | 4466 | { |
|
- | 4467 | int i; |
|
- | 4468 | ||
- | 4469 | for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) |
|
- | 4470 | i915_gem_free_phys_object(dev, i); |
|
- | 4471 | } |
|
- | 4472 | ||
- | 4473 | void i915_gem_detach_phys_object(struct drm_device *dev, |
|
- | 4474 | struct drm_i915_gem_object *obj) |
|
- | 4475 | { |
|
- | 4476 | struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; |
|
- | 4477 | char *vaddr; |
|
- | 4478 | int i; |
|
- | 4479 | int page_count; |
|
- | 4480 | ||
- | 4481 | if (!obj->phys_obj) |
|
- | 4482 | return; |
|
- | 4483 | vaddr = obj->phys_obj->handle->vaddr; |
|
- | 4484 | ||
- | 4485 | page_count = obj->base.size / PAGE_SIZE; |
|
- | 4486 | for (i = 0; i < page_count; i++) { |
|
- | 4487 | struct page *page = shmem_read_mapping_page(mapping, i); |
|
- | 4488 | if (!IS_ERR(page)) { |
|
- | 4489 | char *dst = kmap_atomic(page); |
|
- | 4490 | memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE); |
|
- | 4491 | kunmap_atomic(dst); |
|
- | 4492 | ||
- | 4493 | drm_clflush_pages(&page, 1); |
|
- | 4494 | ||
- | 4495 | set_page_dirty(page); |
|
- | 4496 | mark_page_accessed(page); |
|
- | 4497 | page_cache_release(page); |
|
- | 4498 | } |
|
- | 4499 | } |
|
- | 4500 | i915_gem_chipset_flush(dev); |
|
- | 4501 | ||
- | 4502 | obj->phys_obj->cur_obj = NULL; |
|
- | 4503 | obj->phys_obj = NULL; |
|
- | 4504 | } |
|
- | 4505 | ||
- | 4506 | int |
|
- | 4507 | i915_gem_attach_phys_object(struct drm_device *dev, |
|
- | 4508 | struct drm_i915_gem_object *obj, |
|
- | 4509 | int id, |
|
- | 4510 | int align) |
|
- | 4511 | { |
|
- | 4512 | struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; |
|
- | 4513 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 4514 | int ret = 0; |
|
- | 4515 | int page_count; |
|
- | 4516 | int i; |
|
- | 4517 | ||
- | 4518 | if (id > I915_MAX_PHYS_OBJECT) |
|
- | 4519 | return -EINVAL; |
|
- | 4520 | ||
- | 4521 | if (obj->phys_obj) { |
|
- | 4522 | if (obj->phys_obj->id == id) |
|
- | 4523 | return 0; |
|
- | 4524 | i915_gem_detach_phys_object(dev, obj); |
|
- | 4525 | } |
|
- | 4526 | ||
- | 4527 | /* create a new object */ |
|
- | 4528 | if (!dev_priv->mm.phys_objs[id - 1]) { |
|
- | 4529 | ret = i915_gem_init_phys_object(dev, id, |
|
- | 4530 | obj->base.size, align); |
|
- | 4531 | if (ret) { |
|
- | 4532 | DRM_ERROR("failed to init phys object %d size: %zu\n", |
|
- | 4533 | id, obj->base.size); |
|
- | 4534 | return ret; |
|
- | 4535 | } |
|
- | 4536 | } |
|
- | 4537 | ||
- | 4538 | /* bind to the object */ |
|
- | 4539 | obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; |
|
- | 4540 | obj->phys_obj->cur_obj = obj; |
|
- | 4541 | ||
- | 4542 | page_count = obj->base.size / PAGE_SIZE; |
|
- | 4543 | ||
- | 4544 | for (i = 0; i < page_count; i++) { |
|
- | 4545 | struct page *page; |
|
- | 4546 | char *dst, *src; |
|
- | 4547 | ||
- | 4548 | page = shmem_read_mapping_page(mapping, i); |
|
- | 4549 | if (IS_ERR(page)) |
|
- | 4550 | return PTR_ERR(page); |
|
- | 4551 | ||
- | 4552 | src = kmap_atomic(page); |
|
- | 4553 | dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE); |
|
- | 4554 | memcpy(dst, src, PAGE_SIZE); |
|
- | 4555 | kunmap_atomic(src); |
|
- | 4556 | ||
- | 4557 | mark_page_accessed(page); |
|
- | 4558 | page_cache_release(page); |
|
- | 4559 | } |
|
- | 4560 | ||
- | 4561 | return 0; |
|
- | 4562 | } |
|
- | 4563 | ||
- | 4564 | static int |
|
- | 4565 | i915_gem_phys_pwrite(struct drm_device *dev, |
|
- | 4566 | struct drm_i915_gem_object *obj, |
|
- | 4567 | struct drm_i915_gem_pwrite *args, |
|
- | 4568 | struct drm_file *file_priv) |
|
- | 4569 | { |
|
- | 4570 | void *vaddr = obj->phys_obj->handle->vaddr + args->offset; |
|
- | 4571 | char __user *user_data = to_user_ptr(args->data_ptr); |
|
- | 4572 | ||
- | 4573 | if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { |
|
- | 4574 | unsigned long unwritten; |
|
- | 4575 | ||
- | 4576 | /* The physical object once assigned is fixed for the lifetime |
|
- | 4577 | * of the obj, so we can safely drop the lock and continue |
|
- | 4578 | * to access vaddr. |
|
- | 4579 | */ |
|
- | 4580 | mutex_unlock(&dev->struct_mutex); |
|
- | 4581 | unwritten = copy_from_user(vaddr, user_data, args->size); |
|
- | 4582 | mutex_lock(&dev->struct_mutex); |
|
- | 4583 | if (unwritten) |
|
- | 4584 | return -EFAULT; |
|
- | 4585 | } |
|
- | 4586 | ||
- | 4587 | i915_gem_chipset_flush(dev); |
|
- | 4588 | return 0; |
|
- | 4589 | } |
|
- | 4590 | ||
- | 4591 | void i915_gem_release(struct drm_device *dev, struct drm_file *file) |
|
- | 4592 | { |
|
- | 4593 | struct drm_i915_file_private *file_priv = file->driver_priv; |
|
- | 4594 | ||
- | 4595 | /* Clean up our request list when the client is going away, so that |
|
- | 4596 | * later retire_requests won't dereference our soon-to-be-gone |
|
- | 4597 | * file_priv. |
|
- | 4598 | */ |
|
- | 4599 | spin_lock(&file_priv->mm.lock); |
|
- | 4600 | while (!list_empty(&file_priv->mm.request_list)) { |
|
- | 4601 | struct drm_i915_gem_request *request; |
|
- | 4602 | ||
- | 4603 | request = list_first_entry(&file_priv->mm.request_list, |
|
- | 4604 | struct drm_i915_gem_request, |
|
- | 4605 | client_list); |
|
- | 4606 | list_del(&request->client_list); |
|
- | 4607 | request->file_priv = NULL; |
|
- | 4608 | } |
|
- | 4609 | spin_unlock(&file_priv->mm.lock); |
|
- | 4610 | } |
|
- | 4611 | #endif |
|
- | 4612 | ||
- | 4613 | static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) |
|
- | 4614 | { |
|
- | 4615 | if (!mutex_is_locked(mutex)) |
|
- | 4616 | return false; |
|
- | 4617 | ||
- | 4618 | #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES) |
|
- | 4619 | return mutex->owner == task; |
|
- | 4620 | #else |
|
- | 4621 | /* Since UP may be pre-empted, we cannot assume that we own the lock */ |
|
- | 4622 | return false; |
|
- | 4623 | #endif |
|
- | 4624 | } |
|
- | 4625 | ||
- | 4626 | /* All the new VM stuff */ |
|
- | 4627 | unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o, |
|
- | 4628 | struct i915_address_space *vm) |
|
- | 4629 | { |
|
- | 4630 | struct drm_i915_private *dev_priv = o->base.dev->dev_private; |
|
- | 4631 | struct i915_vma *vma; |
|
- | 4632 | ||
- | 4633 | if (vm == &dev_priv->mm.aliasing_ppgtt->base) |
|
- | 4634 | vm = &dev_priv->gtt.base; |
|
- | 4635 | ||
- | 4636 | BUG_ON(list_empty(&o->vma_list)); |
|
- | 4637 | list_for_each_entry(vma, &o->vma_list, vma_link) { |
|
- | 4638 | if (vma->vm == vm) |
|
- | 4639 | return vma->node.start; |
|
- | 4640 | ||
- | 4641 | } |
|
- | 4642 | return 0; //-1; |
|
- | 4643 | } |
|
- | 4644 | ||
- | 4645 | bool i915_gem_obj_bound(struct drm_i915_gem_object *o, |
|
- | 4646 | struct i915_address_space *vm) |
|
- | 4647 | { |
|
- | 4648 | struct i915_vma *vma; |
|
- | 4649 | ||
- | 4650 | list_for_each_entry(vma, &o->vma_list, vma_link) |
|
- | 4651 | if (vma->vm == vm && drm_mm_node_allocated(&vma->node)) |
|
- | 4652 | return true; |
|
- | 4653 | ||
- | 4654 | return false; |
|
- | 4655 | } |
|
- | 4656 | ||
- | 4657 | bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o) |
|
- | 4658 | { |
|
- | 4659 | struct drm_i915_private *dev_priv = o->base.dev->dev_private; |
|
- | 4660 | struct i915_address_space *vm; |
|
- | 4661 | ||
- | 4662 | list_for_each_entry(vm, &dev_priv->vm_list, global_link) |
|
- | 4663 | if (i915_gem_obj_bound(o, vm)) |
|
- | 4664 | return true; |
|
- | 4665 | ||
- | 4666 | return false; |
|
- | 4667 | } |
|
- | 4668 | ||
- | 4669 | unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, |
|
- | 4670 | struct i915_address_space *vm) |
|
- | 4671 | { |
|
- | 4672 | struct drm_i915_private *dev_priv = o->base.dev->dev_private; |
|
- | 4673 | struct i915_vma *vma; |
|
- | 4674 | ||
- | 4675 | if (vm == &dev_priv->mm.aliasing_ppgtt->base) |
|
- | 4676 | vm = &dev_priv->gtt.base; |
|
- | 4677 | ||
- | 4678 | BUG_ON(list_empty(&o->vma_list)); |
|
- | 4679 | ||
- | 4680 | list_for_each_entry(vma, &o->vma_list, vma_link) |
|
- | 4681 | if (vma->vm == vm) |
|
- | 4682 | return vma->node.size; |
|
- | 4683 | ||
- | 4684 | return 0; |
|
- | 4685 | } |
|
- | 4686 | struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, |
|
- | 4687 | struct i915_address_space *vm) |
|
- | 4688 | { |
|
Line -... | Line 4689... | ||
- | 4689 | struct i915_vma *vma; |
|
- | 4690 | list_for_each_entry(vma, &obj->vma_list, vma_link) |
|
- | 4691 | if (vma->vm == vm) |
|
- | 4692 | return vma; |
|
- | 4693 | ||
- | 4694 | return NULL; |
|
- | 4695 | } |
|
- | 4696 | ||
- | 4697 | struct i915_vma * |
|
- | 4698 | i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, |
|
- | 4699 | struct i915_address_space *vm) |
|
- | 4700 | { |
|
- | 4701 | struct i915_vma *vma; |
|
- | 4702 | ||
- | 4703 | vma = i915_gem_obj_to_vma(obj, vm); |