Rev 3290 | Rev 3480 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 3290 | Rev 3298 | ||
---|---|---|---|
1 | /* |
1 | /* |
2 | * Copyright © 2008 Intel Corporation |
2 | * Copyright © 2008 Intel Corporation |
3 | * |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
13 | * Software. |
14 | * |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
21 | * IN THE SOFTWARE. |
22 | * |
22 | * |
23 | * Authors: |
23 | * Authors: |
24 | * Eric Anholt |
24 | * Eric Anholt |
25 | * |
25 | * |
26 | */ |
26 | */ |
27 | 27 | ||
28 | #include |
28 | #include |
29 | #include |
29 | #include |
30 | #include "i915_drv.h" |
30 | #include "i915_drv.h" |
31 | #include "i915_trace.h" |
31 | #include "i915_trace.h" |
32 | #include "intel_drv.h" |
32 | #include "intel_drv.h" |
33 | #include |
33 | #include |
34 | #include |
34 | #include |
35 | //#include |
35 | //#include |
36 | #include |
36 | #include |
37 | 37 | ||
38 | extern int x86_clflush_size; |
38 | extern int x86_clflush_size; |
39 | 39 | ||
40 | #define PROT_READ 0x1 /* page can be read */ |
40 | #define PROT_READ 0x1 /* page can be read */ |
41 | #define PROT_WRITE 0x2 /* page can be written */ |
41 | #define PROT_WRITE 0x2 /* page can be written */ |
42 | #define MAP_SHARED 0x01 /* Share changes */ |
42 | #define MAP_SHARED 0x01 /* Share changes */ |
43 | 43 | ||
44 | #undef mb |
44 | #undef mb |
45 | #undef rmb |
45 | #undef rmb |
46 | #undef wmb |
46 | #undef wmb |
47 | #define mb() asm volatile("mfence") |
47 | #define mb() asm volatile("mfence") |
48 | #define rmb() asm volatile ("lfence") |
48 | #define rmb() asm volatile ("lfence") |
49 | #define wmb() asm volatile ("sfence") |
49 | #define wmb() asm volatile ("sfence") |
50 | 50 | ||
51 | struct drm_i915_gem_object *get_fb_obj(); |
51 | struct drm_i915_gem_object *get_fb_obj(); |
52 | 52 | ||
53 | unsigned long vm_mmap(struct file *file, unsigned long addr, |
53 | unsigned long vm_mmap(struct file *file, unsigned long addr, |
54 | unsigned long len, unsigned long prot, |
54 | unsigned long len, unsigned long prot, |
55 | unsigned long flag, unsigned long offset); |
55 | unsigned long flag, unsigned long offset); |
56 | 56 | ||
57 | static inline void clflush(volatile void *__p) |
57 | static inline void clflush(volatile void *__p) |
58 | { |
58 | { |
59 | asm volatile("clflush %0" : "+m" (*(volatile char*)__p)); |
59 | asm volatile("clflush %0" : "+m" (*(volatile char*)__p)); |
60 | } |
60 | } |
61 | 61 | ||
62 | #define MAX_ERRNO 4095 |
62 | #define MAX_ERRNO 4095 |
63 | 63 | ||
64 | #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO) |
64 | #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO) |
65 | 65 | ||
66 | 66 | ||
67 | #define I915_EXEC_CONSTANTS_MASK (3<<6) |
67 | #define I915_EXEC_CONSTANTS_MASK (3<<6) |
68 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
68 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
69 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
69 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
70 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
70 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
71 | 71 | ||
72 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
72 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
73 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
73 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
74 | static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
74 | static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
75 | unsigned alignment, |
75 | unsigned alignment, |
76 | bool map_and_fenceable, |
76 | bool map_and_fenceable, |
77 | bool nonblocking); |
77 | bool nonblocking); |
78 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
78 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
79 | struct drm_i915_gem_object *obj, |
79 | struct drm_i915_gem_object *obj, |
80 | struct drm_i915_gem_pwrite *args, |
80 | struct drm_i915_gem_pwrite *args, |
81 | struct drm_file *file); |
81 | struct drm_file *file); |
82 | 82 | ||
83 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
83 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
84 | struct drm_i915_gem_object *obj); |
84 | struct drm_i915_gem_object *obj); |
85 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
85 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
86 | struct drm_i915_fence_reg *fence, |
86 | struct drm_i915_fence_reg *fence, |
87 | bool enable); |
87 | bool enable); |
88 | 88 | ||
89 | static long i915_gem_purge(struct drm_i915_private *dev_priv, long target); |
89 | static long i915_gem_purge(struct drm_i915_private *dev_priv, long target); |
90 | static void i915_gem_shrink_all(struct drm_i915_private *dev_priv); |
90 | static void i915_gem_shrink_all(struct drm_i915_private *dev_priv); |
91 | static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); |
91 | static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); |
92 | 92 | ||
93 | static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) |
93 | static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) |
94 | { |
94 | { |
95 | if (obj->tiling_mode) |
95 | if (obj->tiling_mode) |
96 | i915_gem_release_mmap(obj); |
96 | i915_gem_release_mmap(obj); |
97 | 97 | ||
98 | /* As we do not have an associated fence register, we will force |
98 | /* As we do not have an associated fence register, we will force |
99 | * a tiling change if we ever need to acquire one. |
99 | * a tiling change if we ever need to acquire one. |
100 | */ |
100 | */ |
101 | obj->fence_dirty = false; |
101 | obj->fence_dirty = false; |
102 | obj->fence_reg = I915_FENCE_REG_NONE; |
102 | obj->fence_reg = I915_FENCE_REG_NONE; |
103 | } |
103 | } |
104 | 104 | ||
105 | /* some bookkeeping */ |
105 | /* some bookkeeping */ |
106 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
106 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
107 | size_t size) |
107 | size_t size) |
108 | { |
108 | { |
109 | dev_priv->mm.object_count++; |
109 | dev_priv->mm.object_count++; |
110 | dev_priv->mm.object_memory += size; |
110 | dev_priv->mm.object_memory += size; |
111 | } |
111 | } |
112 | 112 | ||
113 | static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, |
113 | static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, |
114 | size_t size) |
114 | size_t size) |
115 | { |
115 | { |
116 | dev_priv->mm.object_count--; |
116 | dev_priv->mm.object_count--; |
117 | dev_priv->mm.object_memory -= size; |
117 | dev_priv->mm.object_memory -= size; |
118 | } |
118 | } |
119 | 119 | ||
120 | static int |
120 | static int |
121 | i915_gem_wait_for_error(struct drm_device *dev) |
121 | i915_gem_wait_for_error(struct drm_device *dev) |
122 | { |
122 | { |
123 | struct drm_i915_private *dev_priv = dev->dev_private; |
123 | struct drm_i915_private *dev_priv = dev->dev_private; |
124 | struct completion *x = &dev_priv->error_completion; |
124 | struct completion *x = &dev_priv->error_completion; |
125 | unsigned long flags; |
125 | unsigned long flags; |
126 | int ret; |
126 | int ret; |
127 | 127 | ||
128 | if (!atomic_read(&dev_priv->mm.wedged)) |
128 | if (!atomic_read(&dev_priv->mm.wedged)) |
129 | return 0; |
129 | return 0; |
130 | #if 0 |
130 | #if 0 |
131 | /* |
131 | /* |
132 | * Only wait 10 seconds for the gpu reset to complete to avoid hanging |
132 | * Only wait 10 seconds for the gpu reset to complete to avoid hanging |
133 | * userspace. If it takes that long something really bad is going on and |
133 | * userspace. If it takes that long something really bad is going on and |
134 | * we should simply try to bail out and fail as gracefully as possible. |
134 | * we should simply try to bail out and fail as gracefully as possible. |
135 | */ |
135 | */ |
136 | ret = wait_for_completion_interruptible_timeout(x, 10*HZ); |
136 | ret = wait_for_completion_interruptible_timeout(x, 10*HZ); |
137 | if (ret == 0) { |
137 | if (ret == 0) { |
138 | DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); |
138 | DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); |
139 | return -EIO; |
139 | return -EIO; |
140 | } else if (ret < 0) { |
140 | } else if (ret < 0) { |
141 | return ret; |
141 | return ret; |
142 | } |
142 | } |
143 | 143 | ||
144 | if (atomic_read(&dev_priv->mm.wedged)) { |
144 | if (atomic_read(&dev_priv->mm.wedged)) { |
145 | /* GPU is hung, bump the completion count to account for |
145 | /* GPU is hung, bump the completion count to account for |
146 | * the token we just consumed so that we never hit zero and |
146 | * the token we just consumed so that we never hit zero and |
147 | * end up waiting upon a subsequent completion event that |
147 | * end up waiting upon a subsequent completion event that |
148 | * will never happen. |
148 | * will never happen. |
149 | */ |
149 | */ |
150 | spin_lock_irqsave(&x->wait.lock, flags); |
150 | spin_lock_irqsave(&x->wait.lock, flags); |
151 | x->done++; |
151 | x->done++; |
152 | spin_unlock_irqrestore(&x->wait.lock, flags); |
152 | spin_unlock_irqrestore(&x->wait.lock, flags); |
153 | } |
153 | } |
154 | #endif |
154 | #endif |
155 | 155 | ||
156 | return 0; |
156 | return 0; |
157 | } |
157 | } |
158 | 158 | ||
159 | int i915_mutex_lock_interruptible(struct drm_device *dev) |
159 | int i915_mutex_lock_interruptible(struct drm_device *dev) |
160 | { |
160 | { |
161 | int ret; |
161 | int ret; |
162 | 162 | ||
163 | ret = i915_gem_wait_for_error(dev); |
163 | ret = i915_gem_wait_for_error(dev); |
164 | if (ret) |
164 | if (ret) |
165 | return ret; |
165 | return ret; |
166 | 166 | ||
167 | mutex_lock(&dev->struct_mutex); |
167 | mutex_lock(&dev->struct_mutex); |
168 | 168 | ||
169 | WARN_ON(i915_verify_lists(dev)); |
169 | WARN_ON(i915_verify_lists(dev)); |
170 | return 0; |
170 | return 0; |
171 | } |
171 | } |
172 | 172 | ||
173 | static inline bool |
173 | static inline bool |
174 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
174 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
175 | { |
175 | { |
176 | return obj->gtt_space && !obj->active; |
176 | return obj->gtt_space && !obj->active; |
177 | } |
177 | } |
178 | 178 | ||
179 | 179 | ||
180 | #if 0 |
180 | #if 0 |
181 | 181 | ||
182 | int |
182 | int |
183 | i915_gem_init_ioctl(struct drm_device *dev, void *data, |
183 | i915_gem_init_ioctl(struct drm_device *dev, void *data, |
184 | struct drm_file *file) |
184 | struct drm_file *file) |
185 | { |
185 | { |
186 | struct drm_i915_gem_init *args = data; |
186 | struct drm_i915_gem_init *args = data; |
187 | 187 | ||
188 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
188 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
189 | return -ENODEV; |
189 | return -ENODEV; |
190 | 190 | ||
191 | if (args->gtt_start >= args->gtt_end || |
191 | if (args->gtt_start >= args->gtt_end || |
192 | (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) |
192 | (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) |
193 | return -EINVAL; |
193 | return -EINVAL; |
194 | 194 | ||
195 | /* GEM with user mode setting was never supported on ilk and later. */ |
195 | /* GEM with user mode setting was never supported on ilk and later. */ |
196 | if (INTEL_INFO(dev)->gen >= 5) |
196 | if (INTEL_INFO(dev)->gen >= 5) |
197 | return -ENODEV; |
197 | return -ENODEV; |
198 | 198 | ||
199 | mutex_lock(&dev->struct_mutex); |
199 | mutex_lock(&dev->struct_mutex); |
200 | i915_gem_init_global_gtt(dev, args->gtt_start, |
200 | i915_gem_init_global_gtt(dev, args->gtt_start, |
201 | args->gtt_end, args->gtt_end); |
201 | args->gtt_end, args->gtt_end); |
202 | mutex_unlock(&dev->struct_mutex); |
202 | mutex_unlock(&dev->struct_mutex); |
203 | 203 | ||
204 | return 0; |
204 | return 0; |
205 | } |
205 | } |
206 | #endif |
206 | #endif |
207 | 207 | ||
208 | int |
208 | int |
209 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, |
209 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, |
210 | struct drm_file *file) |
210 | struct drm_file *file) |
211 | { |
211 | { |
212 | struct drm_i915_private *dev_priv = dev->dev_private; |
212 | struct drm_i915_private *dev_priv = dev->dev_private; |
213 | struct drm_i915_gem_get_aperture *args = data; |
213 | struct drm_i915_gem_get_aperture *args = data; |
214 | struct drm_i915_gem_object *obj; |
214 | struct drm_i915_gem_object *obj; |
215 | size_t pinned; |
215 | size_t pinned; |
216 | 216 | ||
217 | pinned = 0; |
217 | pinned = 0; |
218 | mutex_lock(&dev->struct_mutex); |
218 | mutex_lock(&dev->struct_mutex); |
219 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) |
219 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) |
220 | if (obj->pin_count) |
220 | if (obj->pin_count) |
221 | pinned += obj->gtt_space->size; |
221 | pinned += obj->gtt_space->size; |
222 | mutex_unlock(&dev->struct_mutex); |
222 | mutex_unlock(&dev->struct_mutex); |
223 | 223 | ||
224 | args->aper_size = dev_priv->mm.gtt_total; |
224 | args->aper_size = dev_priv->mm.gtt_total; |
225 | args->aper_available_size = args->aper_size - pinned; |
225 | args->aper_available_size = args->aper_size - pinned; |
226 | 226 | ||
227 | return 0; |
227 | return 0; |
228 | } |
228 | } |
229 | 229 | ||
230 | static int |
230 | static int |
231 | i915_gem_create(struct drm_file *file, |
231 | i915_gem_create(struct drm_file *file, |
232 | struct drm_device *dev, |
232 | struct drm_device *dev, |
233 | uint64_t size, |
233 | uint64_t size, |
234 | uint32_t *handle_p) |
234 | uint32_t *handle_p) |
235 | { |
235 | { |
236 | struct drm_i915_gem_object *obj; |
236 | struct drm_i915_gem_object *obj; |
237 | int ret; |
237 | int ret; |
238 | u32 handle; |
238 | u32 handle; |
239 | - | ||
240 | ENTER(); |
- | |
241 | 239 | ||
242 | size = roundup(size, PAGE_SIZE); |
240 | size = roundup(size, PAGE_SIZE); |
243 | if (size == 0) |
241 | if (size == 0) |
244 | return -EINVAL; |
242 | return -EINVAL; |
245 | 243 | ||
246 | /* Allocate the new object */ |
244 | /* Allocate the new object */ |
247 | obj = i915_gem_alloc_object(dev, size); |
245 | obj = i915_gem_alloc_object(dev, size); |
248 | if (obj == NULL) |
246 | if (obj == NULL) |
249 | return -ENOMEM; |
247 | return -ENOMEM; |
250 | 248 | ||
251 | ret = drm_gem_handle_create(file, &obj->base, &handle); |
249 | ret = drm_gem_handle_create(file, &obj->base, &handle); |
252 | if (ret) { |
250 | if (ret) { |
253 | drm_gem_object_release(&obj->base); |
251 | drm_gem_object_release(&obj->base); |
254 | i915_gem_info_remove_obj(dev->dev_private, obj->base.size); |
252 | i915_gem_info_remove_obj(dev->dev_private, obj->base.size); |
255 | kfree(obj); |
253 | kfree(obj); |
256 | return ret; |
254 | return ret; |
257 | } |
255 | } |
258 | 256 | ||
259 | /* drop reference from allocate - handle holds it now */ |
257 | /* drop reference from allocate - handle holds it now */ |
260 | drm_gem_object_unreference(&obj->base); |
258 | drm_gem_object_unreference(&obj->base); |
261 | trace_i915_gem_object_create(obj); |
259 | trace_i915_gem_object_create(obj); |
262 | 260 | ||
263 | *handle_p = handle; |
261 | *handle_p = handle; |
264 | 262 | ||
265 | printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle); |
263 | // printf("%s obj %p handle %d\n", __FUNCTION__, obj, handle); |
266 | 264 | ||
267 | return 0; |
265 | return 0; |
268 | } |
266 | } |
269 | 267 | ||
270 | int |
268 | int |
271 | i915_gem_dumb_create(struct drm_file *file, |
269 | i915_gem_dumb_create(struct drm_file *file, |
272 | struct drm_device *dev, |
270 | struct drm_device *dev, |
273 | struct drm_mode_create_dumb *args) |
271 | struct drm_mode_create_dumb *args) |
274 | { |
272 | { |
275 | /* have to work out size/pitch and return them */ |
273 | /* have to work out size/pitch and return them */ |
276 | args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64); |
274 | args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64); |
277 | args->size = args->pitch * args->height; |
275 | args->size = args->pitch * args->height; |
278 | return i915_gem_create(file, dev, |
276 | return i915_gem_create(file, dev, |
279 | args->size, &args->handle); |
277 | args->size, &args->handle); |
280 | } |
278 | } |
281 | 279 | ||
282 | int i915_gem_dumb_destroy(struct drm_file *file, |
280 | int i915_gem_dumb_destroy(struct drm_file *file, |
283 | struct drm_device *dev, |
281 | struct drm_device *dev, |
284 | uint32_t handle) |
282 | uint32_t handle) |
285 | { |
283 | { |
286 | return drm_gem_handle_delete(file, handle); |
284 | return drm_gem_handle_delete(file, handle); |
287 | } |
285 | } |
288 | 286 | ||
289 | /** |
287 | /** |
290 | * Creates a new mm object and returns a handle to it. |
288 | * Creates a new mm object and returns a handle to it. |
291 | */ |
289 | */ |
292 | int |
290 | int |
293 | i915_gem_create_ioctl(struct drm_device *dev, void *data, |
291 | i915_gem_create_ioctl(struct drm_device *dev, void *data, |
294 | struct drm_file *file) |
292 | struct drm_file *file) |
295 | { |
293 | { |
296 | struct drm_i915_gem_create *args = data; |
294 | struct drm_i915_gem_create *args = data; |
297 | 295 | ||
298 | return i915_gem_create(file, dev, |
296 | return i915_gem_create(file, dev, |
299 | args->size, &args->handle); |
297 | args->size, &args->handle); |
300 | } |
298 | } |
301 | 299 | ||
302 | static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) |
300 | static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) |
303 | { |
301 | { |
304 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
302 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
305 | 303 | ||
306 | return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && |
304 | return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && |
307 | obj->tiling_mode != I915_TILING_NONE; |
305 | obj->tiling_mode != I915_TILING_NONE; |
308 | } |
306 | } |
309 | #if 0 |
307 | #if 0 |
310 | 308 | ||
311 | static inline int |
309 | static inline int |
312 | __copy_to_user_swizzled(char __user *cpu_vaddr, |
310 | __copy_to_user_swizzled(char __user *cpu_vaddr, |
313 | const char *gpu_vaddr, int gpu_offset, |
311 | const char *gpu_vaddr, int gpu_offset, |
314 | int length) |
312 | int length) |
315 | { |
313 | { |
316 | int ret, cpu_offset = 0; |
314 | int ret, cpu_offset = 0; |
317 | 315 | ||
318 | while (length > 0) { |
316 | while (length > 0) { |
319 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
317 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
320 | int this_length = min(cacheline_end - gpu_offset, length); |
318 | int this_length = min(cacheline_end - gpu_offset, length); |
321 | int swizzled_gpu_offset = gpu_offset ^ 64; |
319 | int swizzled_gpu_offset = gpu_offset ^ 64; |
322 | 320 | ||
323 | ret = __copy_to_user(cpu_vaddr + cpu_offset, |
321 | ret = __copy_to_user(cpu_vaddr + cpu_offset, |
324 | gpu_vaddr + swizzled_gpu_offset, |
322 | gpu_vaddr + swizzled_gpu_offset, |
325 | this_length); |
323 | this_length); |
326 | if (ret) |
324 | if (ret) |
327 | return ret + length; |
325 | return ret + length; |
328 | 326 | ||
329 | cpu_offset += this_length; |
327 | cpu_offset += this_length; |
330 | gpu_offset += this_length; |
328 | gpu_offset += this_length; |
331 | length -= this_length; |
329 | length -= this_length; |
332 | } |
330 | } |
333 | 331 | ||
334 | return 0; |
332 | return 0; |
335 | } |
333 | } |
336 | 334 | ||
337 | static inline int |
335 | static inline int |
338 | __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, |
336 | __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, |
339 | const char __user *cpu_vaddr, |
337 | const char __user *cpu_vaddr, |
340 | int length) |
338 | int length) |
341 | { |
339 | { |
342 | int ret, cpu_offset = 0; |
340 | int ret, cpu_offset = 0; |
343 | 341 | ||
344 | while (length > 0) { |
342 | while (length > 0) { |
345 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
343 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
346 | int this_length = min(cacheline_end - gpu_offset, length); |
344 | int this_length = min(cacheline_end - gpu_offset, length); |
347 | int swizzled_gpu_offset = gpu_offset ^ 64; |
345 | int swizzled_gpu_offset = gpu_offset ^ 64; |
348 | 346 | ||
349 | ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, |
347 | ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, |
350 | cpu_vaddr + cpu_offset, |
348 | cpu_vaddr + cpu_offset, |
351 | this_length); |
349 | this_length); |
352 | if (ret) |
350 | if (ret) |
353 | return ret + length; |
351 | return ret + length; |
354 | 352 | ||
355 | cpu_offset += this_length; |
353 | cpu_offset += this_length; |
356 | gpu_offset += this_length; |
354 | gpu_offset += this_length; |
357 | length -= this_length; |
355 | length -= this_length; |
358 | } |
356 | } |
359 | 357 | ||
360 | return 0; |
358 | return 0; |
361 | } |
359 | } |
362 | 360 | ||
363 | /* Per-page copy function for the shmem pread fastpath. |
361 | /* Per-page copy function for the shmem pread fastpath. |
364 | * Flushes invalid cachelines before reading the target if |
362 | * Flushes invalid cachelines before reading the target if |
365 | * needs_clflush is set. */ |
363 | * needs_clflush is set. */ |
366 | static int |
364 | static int |
367 | shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, |
365 | shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, |
368 | char __user *user_data, |
366 | char __user *user_data, |
369 | bool page_do_bit17_swizzling, bool needs_clflush) |
367 | bool page_do_bit17_swizzling, bool needs_clflush) |
370 | { |
368 | { |
371 | char *vaddr; |
369 | char *vaddr; |
372 | int ret; |
370 | int ret; |
373 | 371 | ||
374 | if (unlikely(page_do_bit17_swizzling)) |
372 | if (unlikely(page_do_bit17_swizzling)) |
375 | return -EINVAL; |
373 | return -EINVAL; |
376 | 374 | ||
377 | vaddr = kmap_atomic(page); |
375 | vaddr = kmap_atomic(page); |
378 | if (needs_clflush) |
376 | if (needs_clflush) |
379 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
377 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
380 | page_length); |
378 | page_length); |
381 | ret = __copy_to_user_inatomic(user_data, |
379 | ret = __copy_to_user_inatomic(user_data, |
382 | vaddr + shmem_page_offset, |
380 | vaddr + shmem_page_offset, |
383 | page_length); |
381 | page_length); |
384 | kunmap_atomic(vaddr); |
382 | kunmap_atomic(vaddr); |
385 | 383 | ||
386 | return ret ? -EFAULT : 0; |
384 | return ret ? -EFAULT : 0; |
387 | } |
385 | } |
388 | 386 | ||
389 | static void |
387 | static void |
390 | shmem_clflush_swizzled_range(char *addr, unsigned long length, |
388 | shmem_clflush_swizzled_range(char *addr, unsigned long length, |
391 | bool swizzled) |
389 | bool swizzled) |
392 | { |
390 | { |
393 | if (unlikely(swizzled)) { |
391 | if (unlikely(swizzled)) { |
394 | unsigned long start = (unsigned long) addr; |
392 | unsigned long start = (unsigned long) addr; |
395 | unsigned long end = (unsigned long) addr + length; |
393 | unsigned long end = (unsigned long) addr + length; |
396 | 394 | ||
397 | /* For swizzling simply ensure that we always flush both |
395 | /* For swizzling simply ensure that we always flush both |
398 | * channels. Lame, but simple and it works. Swizzled |
396 | * channels. Lame, but simple and it works. Swizzled |
399 | * pwrite/pread is far from a hotpath - current userspace |
397 | * pwrite/pread is far from a hotpath - current userspace |
400 | * doesn't use it at all. */ |
398 | * doesn't use it at all. */ |
401 | start = round_down(start, 128); |
399 | start = round_down(start, 128); |
402 | end = round_up(end, 128); |
400 | end = round_up(end, 128); |
403 | 401 | ||
404 | drm_clflush_virt_range((void *)start, end - start); |
402 | drm_clflush_virt_range((void *)start, end - start); |
405 | } else { |
403 | } else { |
406 | drm_clflush_virt_range(addr, length); |
404 | drm_clflush_virt_range(addr, length); |
407 | } |
405 | } |
408 | 406 | ||
409 | } |
407 | } |
410 | 408 | ||
411 | /* Only difference to the fast-path function is that this can handle bit17 |
409 | /* Only difference to the fast-path function is that this can handle bit17 |
412 | * and uses non-atomic copy and kmap functions. */ |
410 | * and uses non-atomic copy and kmap functions. */ |
413 | static int |
411 | static int |
414 | shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, |
412 | shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, |
415 | char __user *user_data, |
413 | char __user *user_data, |
416 | bool page_do_bit17_swizzling, bool needs_clflush) |
414 | bool page_do_bit17_swizzling, bool needs_clflush) |
417 | { |
415 | { |
418 | char *vaddr; |
416 | char *vaddr; |
419 | int ret; |
417 | int ret; |
420 | 418 | ||
421 | vaddr = kmap(page); |
419 | vaddr = kmap(page); |
422 | if (needs_clflush) |
420 | if (needs_clflush) |
423 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
421 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
424 | page_length, |
422 | page_length, |
425 | page_do_bit17_swizzling); |
423 | page_do_bit17_swizzling); |
426 | 424 | ||
427 | if (page_do_bit17_swizzling) |
425 | if (page_do_bit17_swizzling) |
428 | ret = __copy_to_user_swizzled(user_data, |
426 | ret = __copy_to_user_swizzled(user_data, |
429 | vaddr, shmem_page_offset, |
427 | vaddr, shmem_page_offset, |
430 | page_length); |
428 | page_length); |
431 | else |
429 | else |
432 | ret = __copy_to_user(user_data, |
430 | ret = __copy_to_user(user_data, |
433 | vaddr + shmem_page_offset, |
431 | vaddr + shmem_page_offset, |
434 | page_length); |
432 | page_length); |
435 | kunmap(page); |
433 | kunmap(page); |
436 | 434 | ||
437 | return ret ? - EFAULT : 0; |
435 | return ret ? - EFAULT : 0; |
438 | } |
436 | } |
439 | 437 | ||
440 | static int |
438 | static int |
441 | i915_gem_shmem_pread(struct drm_device *dev, |
439 | i915_gem_shmem_pread(struct drm_device *dev, |
442 | struct drm_i915_gem_object *obj, |
440 | struct drm_i915_gem_object *obj, |
443 | struct drm_i915_gem_pread *args, |
441 | struct drm_i915_gem_pread *args, |
444 | struct drm_file *file) |
442 | struct drm_file *file) |
445 | { |
443 | { |
446 | char __user *user_data; |
444 | char __user *user_data; |
447 | ssize_t remain; |
445 | ssize_t remain; |
448 | loff_t offset; |
446 | loff_t offset; |
449 | int shmem_page_offset, page_length, ret = 0; |
447 | int shmem_page_offset, page_length, ret = 0; |
450 | int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
448 | int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
451 | int hit_slowpath = 0; |
449 | int hit_slowpath = 0; |
452 | int prefaulted = 0; |
450 | int prefaulted = 0; |
453 | int needs_clflush = 0; |
451 | int needs_clflush = 0; |
454 | struct scatterlist *sg; |
452 | struct scatterlist *sg; |
455 | int i; |
453 | int i; |
456 | 454 | ||
457 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
455 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
458 | remain = args->size; |
456 | remain = args->size; |
459 | 457 | ||
460 | obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
458 | obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
461 | 459 | ||
462 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { |
460 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { |
463 | /* If we're not in the cpu read domain, set ourself into the gtt |
461 | /* If we're not in the cpu read domain, set ourself into the gtt |
464 | * read domain and manually flush cachelines (if required). This |
462 | * read domain and manually flush cachelines (if required). This |
465 | * optimizes for the case when the gpu will dirty the data |
463 | * optimizes for the case when the gpu will dirty the data |
466 | * anyway again before the next pread happens. */ |
464 | * anyway again before the next pread happens. */ |
467 | if (obj->cache_level == I915_CACHE_NONE) |
465 | if (obj->cache_level == I915_CACHE_NONE) |
468 | needs_clflush = 1; |
466 | needs_clflush = 1; |
469 | if (obj->gtt_space) { |
467 | if (obj->gtt_space) { |
470 | ret = i915_gem_object_set_to_gtt_domain(obj, false); |
468 | ret = i915_gem_object_set_to_gtt_domain(obj, false); |
471 | if (ret) |
469 | if (ret) |
472 | return ret; |
470 | return ret; |
473 | } |
471 | } |
474 | } |
472 | } |
475 | 473 | ||
476 | ret = i915_gem_object_get_pages(obj); |
474 | ret = i915_gem_object_get_pages(obj); |
477 | if (ret) |
475 | if (ret) |
478 | return ret; |
476 | return ret; |
479 | 477 | ||
480 | i915_gem_object_pin_pages(obj); |
478 | i915_gem_object_pin_pages(obj); |
481 | 479 | ||
482 | offset = args->offset; |
480 | offset = args->offset; |
483 | 481 | ||
484 | for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
482 | for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
485 | struct page *page; |
483 | struct page *page; |
486 | 484 | ||
487 | if (i < offset >> PAGE_SHIFT) |
485 | if (i < offset >> PAGE_SHIFT) |
488 | continue; |
486 | continue; |
489 | 487 | ||
490 | if (remain <= 0) |
488 | if (remain <= 0) |
491 | break; |
489 | break; |
492 | 490 | ||
493 | /* Operation in this page |
491 | /* Operation in this page |
494 | * |
492 | * |
495 | * shmem_page_offset = offset within page in shmem file |
493 | * shmem_page_offset = offset within page in shmem file |
496 | * page_length = bytes to copy for this page |
494 | * page_length = bytes to copy for this page |
497 | */ |
495 | */ |
498 | shmem_page_offset = offset_in_page(offset); |
496 | shmem_page_offset = offset_in_page(offset); |
499 | page_length = remain; |
497 | page_length = remain; |
500 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
498 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
501 | page_length = PAGE_SIZE - shmem_page_offset; |
499 | page_length = PAGE_SIZE - shmem_page_offset; |
502 | 500 | ||
503 | page = sg_page(sg); |
501 | page = sg_page(sg); |
504 | page_do_bit17_swizzling = obj_do_bit17_swizzling && |
502 | page_do_bit17_swizzling = obj_do_bit17_swizzling && |
505 | (page_to_phys(page) & (1 << 17)) != 0; |
503 | (page_to_phys(page) & (1 << 17)) != 0; |
506 | 504 | ||
507 | ret = shmem_pread_fast(page, shmem_page_offset, page_length, |
505 | ret = shmem_pread_fast(page, shmem_page_offset, page_length, |
508 | user_data, page_do_bit17_swizzling, |
506 | user_data, page_do_bit17_swizzling, |
509 | needs_clflush); |
507 | needs_clflush); |
510 | if (ret == 0) |
508 | if (ret == 0) |
511 | goto next_page; |
509 | goto next_page; |
512 | 510 | ||
513 | hit_slowpath = 1; |
511 | hit_slowpath = 1; |
514 | mutex_unlock(&dev->struct_mutex); |
512 | mutex_unlock(&dev->struct_mutex); |
515 | 513 | ||
516 | if (!prefaulted) { |
514 | if (!prefaulted) { |
517 | ret = fault_in_multipages_writeable(user_data, remain); |
515 | ret = fault_in_multipages_writeable(user_data, remain); |
518 | /* Userspace is tricking us, but we've already clobbered |
516 | /* Userspace is tricking us, but we've already clobbered |
519 | * its pages with the prefault and promised to write the |
517 | * its pages with the prefault and promised to write the |
520 | * data up to the first fault. Hence ignore any errors |
518 | * data up to the first fault. Hence ignore any errors |
521 | * and just continue. */ |
519 | * and just continue. */ |
522 | (void)ret; |
520 | (void)ret; |
523 | prefaulted = 1; |
521 | prefaulted = 1; |
524 | } |
522 | } |
525 | 523 | ||
526 | ret = shmem_pread_slow(page, shmem_page_offset, page_length, |
524 | ret = shmem_pread_slow(page, shmem_page_offset, page_length, |
527 | user_data, page_do_bit17_swizzling, |
525 | user_data, page_do_bit17_swizzling, |
528 | needs_clflush); |
526 | needs_clflush); |
529 | 527 | ||
530 | mutex_lock(&dev->struct_mutex); |
528 | mutex_lock(&dev->struct_mutex); |
531 | 529 | ||
532 | next_page: |
530 | next_page: |
533 | mark_page_accessed(page); |
531 | mark_page_accessed(page); |
534 | 532 | ||
535 | if (ret) |
533 | if (ret) |
536 | goto out; |
534 | goto out; |
537 | 535 | ||
538 | remain -= page_length; |
536 | remain -= page_length; |
539 | user_data += page_length; |
537 | user_data += page_length; |
540 | offset += page_length; |
538 | offset += page_length; |
541 | } |
539 | } |
542 | 540 | ||
543 | out: |
541 | out: |
544 | i915_gem_object_unpin_pages(obj); |
542 | i915_gem_object_unpin_pages(obj); |
545 | 543 | ||
546 | if (hit_slowpath) { |
544 | if (hit_slowpath) { |
547 | /* Fixup: Kill any reinstated backing storage pages */ |
545 | /* Fixup: Kill any reinstated backing storage pages */ |
548 | if (obj->madv == __I915_MADV_PURGED) |
546 | if (obj->madv == __I915_MADV_PURGED) |
549 | i915_gem_object_truncate(obj); |
547 | i915_gem_object_truncate(obj); |
550 | } |
548 | } |
551 | 549 | ||
552 | return ret; |
550 | return ret; |
553 | } |
551 | } |
554 | 552 | ||
555 | /** |
553 | /** |
556 | * Reads data from the object referenced by handle. |
554 | * Reads data from the object referenced by handle. |
557 | * |
555 | * |
558 | * On error, the contents of *data are undefined. |
556 | * On error, the contents of *data are undefined. |
559 | */ |
557 | */ |
560 | int |
558 | int |
561 | i915_gem_pread_ioctl(struct drm_device *dev, void *data, |
559 | i915_gem_pread_ioctl(struct drm_device *dev, void *data, |
562 | struct drm_file *file) |
560 | struct drm_file *file) |
563 | { |
561 | { |
564 | struct drm_i915_gem_pread *args = data; |
562 | struct drm_i915_gem_pread *args = data; |
565 | struct drm_i915_gem_object *obj; |
563 | struct drm_i915_gem_object *obj; |
566 | int ret = 0; |
564 | int ret = 0; |
567 | 565 | ||
568 | if (args->size == 0) |
566 | if (args->size == 0) |
569 | return 0; |
567 | return 0; |
570 | 568 | ||
571 | if (!access_ok(VERIFY_WRITE, |
569 | if (!access_ok(VERIFY_WRITE, |
572 | (char __user *)(uintptr_t)args->data_ptr, |
570 | (char __user *)(uintptr_t)args->data_ptr, |
573 | args->size)) |
571 | args->size)) |
574 | return -EFAULT; |
572 | return -EFAULT; |
575 | 573 | ||
576 | ret = i915_mutex_lock_interruptible(dev); |
574 | ret = i915_mutex_lock_interruptible(dev); |
577 | if (ret) |
575 | if (ret) |
578 | return ret; |
576 | return ret; |
579 | 577 | ||
580 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
578 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
581 | if (&obj->base == NULL) { |
579 | if (&obj->base == NULL) { |
582 | ret = -ENOENT; |
580 | ret = -ENOENT; |
583 | goto unlock; |
581 | goto unlock; |
584 | } |
582 | } |
585 | 583 | ||
586 | /* Bounds check source. */ |
584 | /* Bounds check source. */ |
587 | if (args->offset > obj->base.size || |
585 | if (args->offset > obj->base.size || |
588 | args->size > obj->base.size - args->offset) { |
586 | args->size > obj->base.size - args->offset) { |
589 | ret = -EINVAL; |
587 | ret = -EINVAL; |
590 | goto out; |
588 | goto out; |
591 | } |
589 | } |
592 | 590 | ||
593 | /* prime objects have no backing filp to GEM pread/pwrite |
591 | /* prime objects have no backing filp to GEM pread/pwrite |
594 | * pages from. |
592 | * pages from. |
595 | */ |
593 | */ |
596 | if (!obj->base.filp) { |
594 | if (!obj->base.filp) { |
597 | ret = -EINVAL; |
595 | ret = -EINVAL; |
598 | goto out; |
596 | goto out; |
599 | } |
597 | } |
600 | 598 | ||
601 | trace_i915_gem_object_pread(obj, args->offset, args->size); |
599 | trace_i915_gem_object_pread(obj, args->offset, args->size); |
602 | 600 | ||
603 | ret = i915_gem_shmem_pread(dev, obj, args, file); |
601 | ret = i915_gem_shmem_pread(dev, obj, args, file); |
604 | 602 | ||
605 | out: |
603 | out: |
606 | drm_gem_object_unreference(&obj->base); |
604 | drm_gem_object_unreference(&obj->base); |
607 | unlock: |
605 | unlock: |
608 | mutex_unlock(&dev->struct_mutex); |
606 | mutex_unlock(&dev->struct_mutex); |
609 | return ret; |
607 | return ret; |
610 | } |
608 | } |
611 | 609 | ||
612 | /* This is the fast write path which cannot handle |
610 | /* This is the fast write path which cannot handle |
613 | * page faults in the source data |
611 | * page faults in the source data |
614 | */ |
612 | */ |
615 | 613 | ||
616 | static inline int |
614 | static inline int |
617 | fast_user_write(struct io_mapping *mapping, |
615 | fast_user_write(struct io_mapping *mapping, |
618 | loff_t page_base, int page_offset, |
616 | loff_t page_base, int page_offset, |
619 | char __user *user_data, |
617 | char __user *user_data, |
620 | int length) |
618 | int length) |
621 | { |
619 | { |
622 | void __iomem *vaddr_atomic; |
620 | void __iomem *vaddr_atomic; |
623 | void *vaddr; |
621 | void *vaddr; |
624 | unsigned long unwritten; |
622 | unsigned long unwritten; |
625 | 623 | ||
626 | vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); |
624 | vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); |
627 | /* We can use the cpu mem copy function because this is X86. */ |
625 | /* We can use the cpu mem copy function because this is X86. */ |
628 | vaddr = (void __force*)vaddr_atomic + page_offset; |
626 | vaddr = (void __force*)vaddr_atomic + page_offset; |
629 | unwritten = __copy_from_user_inatomic_nocache(vaddr, |
627 | unwritten = __copy_from_user_inatomic_nocache(vaddr, |
630 | user_data, length); |
628 | user_data, length); |
631 | io_mapping_unmap_atomic(vaddr_atomic); |
629 | io_mapping_unmap_atomic(vaddr_atomic); |
632 | return unwritten; |
630 | return unwritten; |
633 | } |
631 | } |
634 | #endif |
632 | #endif |
635 | 633 | ||
636 | #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) |
634 | #define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) |
637 | /** |
635 | /** |
638 | * This is the fast pwrite path, where we copy the data directly from the |
636 | * This is the fast pwrite path, where we copy the data directly from the |
639 | * user into the GTT, uncached. |
637 | * user into the GTT, uncached. |
640 | */ |
638 | */ |
641 | static int |
639 | static int |
642 | i915_gem_gtt_pwrite_fast(struct drm_device *dev, |
640 | i915_gem_gtt_pwrite_fast(struct drm_device *dev, |
643 | struct drm_i915_gem_object *obj, |
641 | struct drm_i915_gem_object *obj, |
644 | struct drm_i915_gem_pwrite *args, |
642 | struct drm_i915_gem_pwrite *args, |
645 | struct drm_file *file) |
643 | struct drm_file *file) |
646 | { |
644 | { |
647 | drm_i915_private_t *dev_priv = dev->dev_private; |
645 | drm_i915_private_t *dev_priv = dev->dev_private; |
648 | ssize_t remain; |
646 | ssize_t remain; |
649 | loff_t offset, page_base; |
647 | loff_t offset, page_base; |
650 | char __user *user_data; |
648 | char __user *user_data; |
651 | int page_offset, page_length, ret; |
649 | int page_offset, page_length, ret; |
652 | char *vaddr; |
650 | char *vaddr; |
653 | 651 | ||
654 | ret = i915_gem_object_pin(obj, 0, true, true); |
652 | ret = i915_gem_object_pin(obj, 0, true, true); |
655 | if (ret) |
653 | if (ret) |
656 | goto out; |
654 | goto out; |
657 | 655 | ||
658 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
656 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
659 | if (ret) |
657 | if (ret) |
660 | goto out_unpin; |
658 | goto out_unpin; |
661 | 659 | ||
662 | ret = i915_gem_object_put_fence(obj); |
660 | ret = i915_gem_object_put_fence(obj); |
663 | if (ret) |
661 | if (ret) |
664 | goto out_unpin; |
662 | goto out_unpin; |
665 | 663 | ||
666 | vaddr = AllocKernelSpace(4096); |
664 | vaddr = AllocKernelSpace(4096); |
667 | if(vaddr == NULL) |
665 | if(vaddr == NULL) |
668 | { |
666 | { |
669 | ret = -ENOSPC; |
667 | ret = -ENOSPC; |
670 | goto out_unpin; |
668 | goto out_unpin; |
671 | }; |
669 | }; |
672 | 670 | ||
673 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
671 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
674 | remain = args->size; |
672 | remain = args->size; |
675 | 673 | ||
676 | offset = obj->gtt_offset + args->offset; |
674 | offset = obj->gtt_offset + args->offset; |
677 | 675 | ||
678 | while (remain > 0) { |
676 | while (remain > 0) { |
679 | /* Operation in this page |
677 | /* Operation in this page |
680 | * |
678 | * |
681 | * page_base = page offset within aperture |
679 | * page_base = page offset within aperture |
682 | * page_offset = offset within page |
680 | * page_offset = offset within page |
683 | * page_length = bytes to copy for this page |
681 | * page_length = bytes to copy for this page |
684 | */ |
682 | */ |
685 | page_base = offset & PAGE_MASK; |
683 | page_base = offset & PAGE_MASK; |
686 | page_offset = offset_in_page(offset); |
684 | page_offset = offset_in_page(offset); |
687 | page_length = remain; |
685 | page_length = remain; |
688 | if ((page_offset + remain) > PAGE_SIZE) |
686 | if ((page_offset + remain) > PAGE_SIZE) |
689 | page_length = PAGE_SIZE - page_offset; |
687 | page_length = PAGE_SIZE - page_offset; |
690 | 688 | ||
691 | MapPage(vaddr, page_base, PG_SW|PG_NOCACHE); |
689 | MapPage(vaddr, page_base, PG_SW|PG_NOCACHE); |
692 | 690 | ||
693 | memcpy(vaddr+page_offset, user_data, page_length); |
691 | memcpy(vaddr+page_offset, user_data, page_length); |
694 | 692 | ||
695 | remain -= page_length; |
693 | remain -= page_length; |
696 | user_data += page_length; |
694 | user_data += page_length; |
697 | offset += page_length; |
695 | offset += page_length; |
698 | } |
696 | } |
699 | 697 | ||
700 | FreeKernelSpace(vaddr); |
698 | FreeKernelSpace(vaddr); |
701 | 699 | ||
702 | out_unpin: |
700 | out_unpin: |
703 | i915_gem_object_unpin(obj); |
701 | i915_gem_object_unpin(obj); |
704 | out: |
702 | out: |
705 | printf("% s ret = %d\n", __FUNCTION__, ret); |
703 | printf("% s ret = %d\n", __FUNCTION__, ret); |
706 | 704 | ||
707 | return ret; |
705 | return ret; |
708 | } |
706 | } |
709 | 707 | ||
710 | /* Per-page copy function for the shmem pwrite fastpath. |
708 | /* Per-page copy function for the shmem pwrite fastpath. |
711 | * Flushes invalid cachelines before writing to the target if |
709 | * Flushes invalid cachelines before writing to the target if |
712 | * needs_clflush_before is set and flushes out any written cachelines after |
710 | * needs_clflush_before is set and flushes out any written cachelines after |
713 | * writing if needs_clflush is set. */ |
711 | * writing if needs_clflush is set. */ |
714 | static int |
712 | static int |
715 | shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, |
713 | shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, |
716 | char __user *user_data, |
714 | char __user *user_data, |
717 | bool page_do_bit17_swizzling, |
715 | bool page_do_bit17_swizzling, |
718 | bool needs_clflush_before, |
716 | bool needs_clflush_before, |
719 | bool needs_clflush_after) |
717 | bool needs_clflush_after) |
720 | { |
718 | { |
721 | char *vaddr; |
719 | char *vaddr; |
722 | int ret = 0; |
720 | int ret = 0; |
723 | 721 | ||
724 | if (unlikely(page_do_bit17_swizzling)) |
722 | if (unlikely(page_do_bit17_swizzling)) |
725 | return -EINVAL; |
723 | return -EINVAL; |
726 | 724 | ||
727 | vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW); |
725 | vaddr = (char *)MapIoMem((addr_t)page, 4096, PG_SW); |
728 | if (needs_clflush_before) |
726 | if (needs_clflush_before) |
729 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
727 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
730 | page_length); |
728 | page_length); |
731 | memcpy(vaddr + shmem_page_offset, |
729 | memcpy(vaddr + shmem_page_offset, |
732 | user_data, |
730 | user_data, |
733 | page_length); |
731 | page_length); |
734 | if (needs_clflush_after) |
732 | if (needs_clflush_after) |
735 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
733 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
736 | page_length); |
734 | page_length); |
737 | FreeKernelSpace(vaddr); |
735 | FreeKernelSpace(vaddr); |
738 | 736 | ||
739 | return ret ? -EFAULT : 0; |
737 | return ret ? -EFAULT : 0; |
740 | } |
738 | } |
741 | #if 0 |
739 | #if 0 |
742 | 740 | ||
743 | /* Only difference to the fast-path function is that this can handle bit17 |
741 | /* Only difference to the fast-path function is that this can handle bit17 |
744 | * and uses non-atomic copy and kmap functions. */ |
742 | * and uses non-atomic copy and kmap functions. */ |
745 | static int |
743 | static int |
746 | shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, |
744 | shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, |
747 | char __user *user_data, |
745 | char __user *user_data, |
748 | bool page_do_bit17_swizzling, |
746 | bool page_do_bit17_swizzling, |
749 | bool needs_clflush_before, |
747 | bool needs_clflush_before, |
750 | bool needs_clflush_after) |
748 | bool needs_clflush_after) |
751 | { |
749 | { |
752 | char *vaddr; |
750 | char *vaddr; |
753 | int ret; |
751 | int ret; |
754 | 752 | ||
755 | vaddr = kmap(page); |
753 | vaddr = kmap(page); |
756 | if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) |
754 | if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) |
757 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
755 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
758 | page_length, |
756 | page_length, |
759 | page_do_bit17_swizzling); |
757 | page_do_bit17_swizzling); |
760 | if (page_do_bit17_swizzling) |
758 | if (page_do_bit17_swizzling) |
761 | ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, |
759 | ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, |
762 | user_data, |
760 | user_data, |
763 | page_length); |
761 | page_length); |
764 | else |
762 | else |
765 | ret = __copy_from_user(vaddr + shmem_page_offset, |
763 | ret = __copy_from_user(vaddr + shmem_page_offset, |
766 | user_data, |
764 | user_data, |
767 | page_length); |
765 | page_length); |
768 | if (needs_clflush_after) |
766 | if (needs_clflush_after) |
769 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
767 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
770 | page_length, |
768 | page_length, |
771 | page_do_bit17_swizzling); |
769 | page_do_bit17_swizzling); |
772 | kunmap(page); |
770 | kunmap(page); |
773 | 771 | ||
774 | return ret ? -EFAULT : 0; |
772 | return ret ? -EFAULT : 0; |
775 | } |
773 | } |
776 | #endif |
774 | #endif |
777 | 775 | ||
778 | 776 | ||
779 | static int |
777 | static int |
780 | i915_gem_shmem_pwrite(struct drm_device *dev, |
778 | i915_gem_shmem_pwrite(struct drm_device *dev, |
781 | struct drm_i915_gem_object *obj, |
779 | struct drm_i915_gem_object *obj, |
782 | struct drm_i915_gem_pwrite *args, |
780 | struct drm_i915_gem_pwrite *args, |
783 | struct drm_file *file) |
781 | struct drm_file *file) |
784 | { |
782 | { |
785 | ssize_t remain; |
783 | ssize_t remain; |
786 | loff_t offset; |
784 | loff_t offset; |
787 | char __user *user_data; |
785 | char __user *user_data; |
788 | int shmem_page_offset, page_length, ret = 0; |
786 | int shmem_page_offset, page_length, ret = 0; |
789 | int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
787 | int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
790 | int hit_slowpath = 0; |
788 | int hit_slowpath = 0; |
791 | int needs_clflush_after = 0; |
789 | int needs_clflush_after = 0; |
792 | int needs_clflush_before = 0; |
790 | int needs_clflush_before = 0; |
793 | int i; |
791 | int i; |
794 | struct scatterlist *sg; |
792 | struct scatterlist *sg; |
795 | 793 | ||
796 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
794 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
797 | remain = args->size; |
795 | remain = args->size; |
798 | 796 | ||
799 | obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
797 | obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
800 | 798 | ||
801 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
799 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
802 | /* If we're not in the cpu write domain, set ourself into the gtt |
800 | /* If we're not in the cpu write domain, set ourself into the gtt |
803 | * write domain and manually flush cachelines (if required). This |
801 | * write domain and manually flush cachelines (if required). This |
804 | * optimizes for the case when the gpu will use the data |
802 | * optimizes for the case when the gpu will use the data |
805 | * right away and we therefore have to clflush anyway. */ |
803 | * right away and we therefore have to clflush anyway. */ |
806 | if (obj->cache_level == I915_CACHE_NONE) |
804 | if (obj->cache_level == I915_CACHE_NONE) |
807 | needs_clflush_after = 1; |
805 | needs_clflush_after = 1; |
808 | if (obj->gtt_space) { |
806 | if (obj->gtt_space) { |
809 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
807 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
810 | if (ret) |
808 | if (ret) |
811 | return ret; |
809 | return ret; |
812 | } |
810 | } |
813 | } |
811 | } |
814 | /* Same trick applies for invalidate partially written cachelines before |
812 | /* Same trick applies for invalidate partially written cachelines before |
815 | * writing. */ |
813 | * writing. */ |
816 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) |
814 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) |
817 | && obj->cache_level == I915_CACHE_NONE) |
815 | && obj->cache_level == I915_CACHE_NONE) |
818 | needs_clflush_before = 1; |
816 | needs_clflush_before = 1; |
819 | 817 | ||
820 | ret = i915_gem_object_get_pages(obj); |
818 | ret = i915_gem_object_get_pages(obj); |
821 | if (ret) |
819 | if (ret) |
822 | return ret; |
820 | return ret; |
823 | 821 | ||
824 | i915_gem_object_pin_pages(obj); |
822 | i915_gem_object_pin_pages(obj); |
825 | 823 | ||
826 | offset = args->offset; |
824 | offset = args->offset; |
827 | obj->dirty = 1; |
825 | obj->dirty = 1; |
828 | 826 | ||
829 | for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
827 | for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
830 | struct page *page; |
828 | struct page *page; |
831 | int partial_cacheline_write; |
829 | int partial_cacheline_write; |
832 | 830 | ||
833 | if (i < offset >> PAGE_SHIFT) |
831 | if (i < offset >> PAGE_SHIFT) |
834 | continue; |
832 | continue; |
835 | 833 | ||
836 | if (remain <= 0) |
834 | if (remain <= 0) |
837 | break; |
835 | break; |
838 | 836 | ||
839 | /* Operation in this page |
837 | /* Operation in this page |
840 | * |
838 | * |
841 | * shmem_page_offset = offset within page in shmem file |
839 | * shmem_page_offset = offset within page in shmem file |
842 | * page_length = bytes to copy for this page |
840 | * page_length = bytes to copy for this page |
843 | */ |
841 | */ |
844 | shmem_page_offset = offset_in_page(offset); |
842 | shmem_page_offset = offset_in_page(offset); |
845 | 843 | ||
846 | page_length = remain; |
844 | page_length = remain; |
847 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
845 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
848 | page_length = PAGE_SIZE - shmem_page_offset; |
846 | page_length = PAGE_SIZE - shmem_page_offset; |
849 | 847 | ||
850 | /* If we don't overwrite a cacheline completely we need to be |
848 | /* If we don't overwrite a cacheline completely we need to be |
851 | * careful to have up-to-date data by first clflushing. Don't |
849 | * careful to have up-to-date data by first clflushing. Don't |
852 | * overcomplicate things and flush the entire patch. */ |
850 | * overcomplicate things and flush the entire patch. */ |
853 | partial_cacheline_write = needs_clflush_before && |
851 | partial_cacheline_write = needs_clflush_before && |
854 | ((shmem_page_offset | page_length) |
852 | ((shmem_page_offset | page_length) |
855 | & (x86_clflush_size - 1)); |
853 | & (x86_clflush_size - 1)); |
856 | 854 | ||
857 | page = sg_page(sg); |
855 | page = sg_page(sg); |
858 | page_do_bit17_swizzling = obj_do_bit17_swizzling && |
856 | page_do_bit17_swizzling = obj_do_bit17_swizzling && |
859 | (page_to_phys(page) & (1 << 17)) != 0; |
857 | (page_to_phys(page) & (1 << 17)) != 0; |
860 | 858 | ||
861 | ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, |
859 | ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, |
862 | user_data, page_do_bit17_swizzling, |
860 | user_data, page_do_bit17_swizzling, |
863 | partial_cacheline_write, |
861 | partial_cacheline_write, |
864 | needs_clflush_after); |
862 | needs_clflush_after); |
865 | if (ret == 0) |
863 | if (ret == 0) |
866 | goto next_page; |
864 | goto next_page; |
867 | 865 | ||
868 | hit_slowpath = 1; |
866 | hit_slowpath = 1; |
869 | mutex_unlock(&dev->struct_mutex); |
867 | mutex_unlock(&dev->struct_mutex); |
870 | dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__); |
868 | dbgprintf("%s need shmem_pwrite_slow\n",__FUNCTION__); |
871 | 869 | ||
872 | // ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, |
870 | // ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, |
873 | // user_data, page_do_bit17_swizzling, |
871 | // user_data, page_do_bit17_swizzling, |
874 | // partial_cacheline_write, |
872 | // partial_cacheline_write, |
875 | // needs_clflush_after); |
873 | // needs_clflush_after); |
876 | 874 | ||
877 | mutex_lock(&dev->struct_mutex); |
875 | mutex_lock(&dev->struct_mutex); |
878 | 876 | ||
879 | next_page: |
877 | next_page: |
880 | 878 | ||
881 | if (ret) |
879 | if (ret) |
882 | goto out; |
880 | goto out; |
883 | 881 | ||
884 | remain -= page_length; |
882 | remain -= page_length; |
885 | user_data += page_length; |
883 | user_data += page_length; |
886 | offset += page_length; |
884 | offset += page_length; |
887 | } |
885 | } |
888 | 886 | ||
889 | out: |
887 | out: |
890 | i915_gem_object_unpin_pages(obj); |
888 | i915_gem_object_unpin_pages(obj); |
891 | 889 | ||
892 | if (hit_slowpath) { |
890 | if (hit_slowpath) { |
893 | /* Fixup: Kill any reinstated backing storage pages */ |
891 | /* Fixup: Kill any reinstated backing storage pages */ |
894 | if (obj->madv == __I915_MADV_PURGED) |
892 | if (obj->madv == __I915_MADV_PURGED) |
895 | i915_gem_object_truncate(obj); |
893 | i915_gem_object_truncate(obj); |
896 | /* and flush dirty cachelines in case the object isn't in the cpu write |
894 | /* and flush dirty cachelines in case the object isn't in the cpu write |
897 | * domain anymore. */ |
895 | * domain anymore. */ |
898 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
896 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
899 | i915_gem_clflush_object(obj); |
897 | i915_gem_clflush_object(obj); |
900 | i915_gem_chipset_flush(dev); |
898 | i915_gem_chipset_flush(dev); |
901 | } |
899 | } |
902 | } |
900 | } |
903 | 901 | ||
904 | if (needs_clflush_after) |
902 | if (needs_clflush_after) |
905 | i915_gem_chipset_flush(dev); |
903 | i915_gem_chipset_flush(dev); |
906 | 904 | ||
907 | return ret; |
905 | return ret; |
908 | } |
906 | } |
909 | 907 | ||
910 | /** |
908 | /** |
911 | * Writes data to the object referenced by handle. |
909 | * Writes data to the object referenced by handle. |
912 | * |
910 | * |
913 | * On error, the contents of the buffer that were to be modified are undefined. |
911 | * On error, the contents of the buffer that were to be modified are undefined. |
914 | */ |
912 | */ |
915 | int |
913 | int |
916 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, |
914 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, |
917 | struct drm_file *file) |
915 | struct drm_file *file) |
918 | { |
916 | { |
919 | struct drm_i915_gem_pwrite *args = data; |
917 | struct drm_i915_gem_pwrite *args = data; |
920 | struct drm_i915_gem_object *obj; |
918 | struct drm_i915_gem_object *obj; |
921 | int ret; |
919 | int ret; |
922 | 920 | ||
923 | if (args->size == 0) |
921 | if (args->size == 0) |
924 | return 0; |
922 | return 0; |
925 | 923 | ||
926 | ret = i915_mutex_lock_interruptible(dev); |
924 | ret = i915_mutex_lock_interruptible(dev); |
927 | if (ret) |
925 | if (ret) |
928 | return ret; |
926 | return ret; |
929 | 927 | ||
930 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
928 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
931 | if (&obj->base == NULL) { |
929 | if (&obj->base == NULL) { |
932 | ret = -ENOENT; |
930 | ret = -ENOENT; |
933 | goto unlock; |
931 | goto unlock; |
934 | } |
932 | } |
935 | 933 | ||
936 | /* Bounds check destination. */ |
934 | /* Bounds check destination. */ |
937 | if (args->offset > obj->base.size || |
935 | if (args->offset > obj->base.size || |
938 | args->size > obj->base.size - args->offset) { |
936 | args->size > obj->base.size - args->offset) { |
939 | ret = -EINVAL; |
937 | ret = -EINVAL; |
940 | goto out; |
938 | goto out; |
941 | } |
939 | } |
942 | 940 | ||
943 | /* prime objects have no backing filp to GEM pread/pwrite |
941 | /* prime objects have no backing filp to GEM pread/pwrite |
944 | * pages from. |
942 | * pages from. |
945 | */ |
943 | */ |
946 | if (!obj->base.filp) { |
944 | if (!obj->base.filp) { |
947 | ret = -EINVAL; |
945 | ret = -EINVAL; |
948 | goto out; |
946 | goto out; |
949 | } |
947 | } |
950 | 948 | ||
951 | trace_i915_gem_object_pwrite(obj, args->offset, args->size); |
949 | trace_i915_gem_object_pwrite(obj, args->offset, args->size); |
952 | 950 | ||
953 | ret = -EFAULT; |
951 | ret = -EFAULT; |
954 | /* We can only do the GTT pwrite on untiled buffers, as otherwise |
952 | /* We can only do the GTT pwrite on untiled buffers, as otherwise |
955 | * it would end up going through the fenced access, and we'll get |
953 | * it would end up going through the fenced access, and we'll get |
956 | * different detiling behavior between reading and writing. |
954 | * different detiling behavior between reading and writing. |
957 | * pread/pwrite currently are reading and writing from the CPU |
955 | * pread/pwrite currently are reading and writing from the CPU |
958 | * perspective, requiring manual detiling by the client. |
956 | * perspective, requiring manual detiling by the client. |
959 | */ |
957 | */ |
960 | // if (obj->phys_obj) { |
958 | // if (obj->phys_obj) { |
961 | // ret = i915_gem_phys_pwrite(dev, obj, args, file); |
959 | // ret = i915_gem_phys_pwrite(dev, obj, args, file); |
962 | // goto out; |
960 | // goto out; |
963 | // } |
961 | // } |
964 | 962 | ||
965 | if (obj->cache_level == I915_CACHE_NONE && |
963 | if (obj->cache_level == I915_CACHE_NONE && |
966 | obj->tiling_mode == I915_TILING_NONE && |
964 | obj->tiling_mode == I915_TILING_NONE && |
967 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
965 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
968 | ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); |
966 | ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); |
969 | /* Note that the gtt paths might fail with non-page-backed user |
967 | /* Note that the gtt paths might fail with non-page-backed user |
970 | * pointers (e.g. gtt mappings when moving data between |
968 | * pointers (e.g. gtt mappings when moving data between |
971 | * textures). Fallback to the shmem path in that case. */ |
969 | * textures). Fallback to the shmem path in that case. */ |
972 | } |
970 | } |
973 | 971 | ||
974 | if (ret == -EFAULT || ret == -ENOSPC) |
972 | if (ret == -EFAULT || ret == -ENOSPC) |
975 | ret = i915_gem_shmem_pwrite(dev, obj, args, file); |
973 | ret = i915_gem_shmem_pwrite(dev, obj, args, file); |
976 | 974 | ||
977 | out: |
975 | out: |
978 | drm_gem_object_unreference(&obj->base); |
976 | drm_gem_object_unreference(&obj->base); |
979 | unlock: |
977 | unlock: |
980 | mutex_unlock(&dev->struct_mutex); |
978 | mutex_unlock(&dev->struct_mutex); |
981 | return ret; |
979 | return ret; |
982 | } |
980 | } |
983 | 981 | ||
984 | int |
982 | int |
985 | i915_gem_check_wedge(struct drm_i915_private *dev_priv, |
983 | i915_gem_check_wedge(struct drm_i915_private *dev_priv, |
986 | bool interruptible) |
984 | bool interruptible) |
987 | { |
985 | { |
988 | if (atomic_read(&dev_priv->mm.wedged)) { |
986 | if (atomic_read(&dev_priv->mm.wedged)) { |
989 | struct completion *x = &dev_priv->error_completion; |
987 | struct completion *x = &dev_priv->error_completion; |
990 | bool recovery_complete; |
988 | bool recovery_complete; |
991 | unsigned long flags; |
989 | unsigned long flags; |
992 | 990 | ||
993 | /* Give the error handler a chance to run. */ |
991 | /* Give the error handler a chance to run. */ |
994 | spin_lock_irqsave(&x->wait.lock, flags); |
992 | spin_lock_irqsave(&x->wait.lock, flags); |
995 | recovery_complete = x->done > 0; |
993 | recovery_complete = x->done > 0; |
996 | spin_unlock_irqrestore(&x->wait.lock, flags); |
994 | spin_unlock_irqrestore(&x->wait.lock, flags); |
997 | 995 | ||
998 | /* Non-interruptible callers can't handle -EAGAIN, hence return |
996 | /* Non-interruptible callers can't handle -EAGAIN, hence return |
999 | * -EIO unconditionally for these. */ |
997 | * -EIO unconditionally for these. */ |
1000 | if (!interruptible) |
998 | if (!interruptible) |
1001 | return -EIO; |
999 | return -EIO; |
1002 | 1000 | ||
1003 | /* Recovery complete, but still wedged means reset failure. */ |
1001 | /* Recovery complete, but still wedged means reset failure. */ |
1004 | if (recovery_complete) |
1002 | if (recovery_complete) |
1005 | return -EIO; |
1003 | return -EIO; |
1006 | 1004 | ||
1007 | return -EAGAIN; |
1005 | return -EAGAIN; |
1008 | } |
1006 | } |
1009 | 1007 | ||
1010 | return 0; |
1008 | return 0; |
1011 | } |
1009 | } |
1012 | 1010 | ||
1013 | /* |
1011 | /* |
1014 | * Compare seqno against outstanding lazy request. Emit a request if they are |
1012 | * Compare seqno against outstanding lazy request. Emit a request if they are |
1015 | * equal. |
1013 | * equal. |
1016 | */ |
1014 | */ |
1017 | static int |
1015 | static int |
1018 | i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) |
1016 | i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) |
1019 | { |
1017 | { |
1020 | int ret; |
1018 | int ret; |
1021 | 1019 | ||
1022 | BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); |
1020 | BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); |
1023 | 1021 | ||
1024 | ret = 0; |
1022 | ret = 0; |
1025 | if (seqno == ring->outstanding_lazy_request) |
1023 | if (seqno == ring->outstanding_lazy_request) |
1026 | ret = i915_add_request(ring, NULL, NULL); |
1024 | ret = i915_add_request(ring, NULL, NULL); |
1027 | 1025 | ||
1028 | return ret; |
1026 | return ret; |
1029 | } |
1027 | } |
1030 | 1028 | ||
1031 | /** |
1029 | /** |
1032 | * __wait_seqno - wait until execution of seqno has finished |
1030 | * __wait_seqno - wait until execution of seqno has finished |
1033 | * @ring: the ring expected to report seqno |
1031 | * @ring: the ring expected to report seqno |
1034 | * @seqno: duh! |
1032 | * @seqno: duh! |
1035 | * @interruptible: do an interruptible wait (normally yes) |
1033 | * @interruptible: do an interruptible wait (normally yes) |
1036 | * @timeout: in - how long to wait (NULL forever); out - how much time remaining |
1034 | * @timeout: in - how long to wait (NULL forever); out - how much time remaining |
1037 | * |
1035 | * |
1038 | * Returns 0 if the seqno was found within the alloted time. Else returns the |
1036 | * Returns 0 if the seqno was found within the alloted time. Else returns the |
1039 | * errno with remaining time filled in timeout argument. |
1037 | * errno with remaining time filled in timeout argument. |
1040 | */ |
1038 | */ |
1041 | static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, |
1039 | static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, |
1042 | bool interruptible, struct timespec *timeout) |
1040 | bool interruptible, struct timespec *timeout) |
1043 | { |
1041 | { |
1044 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1042 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1045 | struct timespec before, now, wait_time={1,0}; |
1043 | struct timespec before, now, wait_time={1,0}; |
1046 | unsigned long timeout_jiffies; |
1044 | unsigned long timeout_jiffies; |
1047 | long end; |
1045 | long end; |
1048 | bool wait_forever = true; |
1046 | bool wait_forever = true; |
1049 | int ret; |
1047 | int ret; |
1050 | 1048 | ||
1051 | if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) |
1049 | if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) |
1052 | return 0; |
1050 | return 0; |
1053 | 1051 | ||
1054 | trace_i915_gem_request_wait_begin(ring, seqno); |
1052 | trace_i915_gem_request_wait_begin(ring, seqno); |
1055 | 1053 | ||
1056 | if (timeout != NULL) { |
1054 | if (timeout != NULL) { |
1057 | wait_time = *timeout; |
1055 | wait_time = *timeout; |
1058 | wait_forever = false; |
1056 | wait_forever = false; |
1059 | } |
1057 | } |
1060 | 1058 | ||
1061 | timeout_jiffies = timespec_to_jiffies(&wait_time); |
1059 | timeout_jiffies = timespec_to_jiffies(&wait_time); |
1062 | 1060 | ||
1063 | if (WARN_ON(!ring->irq_get(ring))) |
1061 | if (WARN_ON(!ring->irq_get(ring))) |
1064 | return -ENODEV; |
1062 | return -ENODEV; |
1065 | 1063 | ||
1066 | /* Record current time in case interrupted by signal, or wedged * */ |
1064 | /* Record current time in case interrupted by signal, or wedged * */ |
1067 | getrawmonotonic(&before); |
1065 | getrawmonotonic(&before); |
1068 | 1066 | ||
1069 | #define EXIT_COND \ |
1067 | #define EXIT_COND \ |
1070 | (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ |
1068 | (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ |
1071 | atomic_read(&dev_priv->mm.wedged)) |
1069 | atomic_read(&dev_priv->mm.wedged)) |
1072 | do { |
1070 | do { |
1073 | if (interruptible) |
1071 | if (interruptible) |
1074 | end = wait_event_interruptible_timeout(ring->irq_queue, |
1072 | end = wait_event_interruptible_timeout(ring->irq_queue, |
1075 | EXIT_COND, |
1073 | EXIT_COND, |
1076 | timeout_jiffies); |
1074 | timeout_jiffies); |
1077 | else |
1075 | else |
1078 | end = wait_event_timeout(ring->irq_queue, EXIT_COND, |
1076 | end = wait_event_timeout(ring->irq_queue, EXIT_COND, |
1079 | timeout_jiffies); |
1077 | timeout_jiffies); |
1080 | 1078 | ||
1081 | ret = i915_gem_check_wedge(dev_priv, interruptible); |
1079 | ret = i915_gem_check_wedge(dev_priv, interruptible); |
1082 | if (ret) |
1080 | if (ret) |
1083 | end = ret; |
1081 | end = ret; |
1084 | } while (end == 0 && wait_forever); |
1082 | } while (end == 0 && wait_forever); |
1085 | 1083 | ||
1086 | getrawmonotonic(&now); |
1084 | getrawmonotonic(&now); |
1087 | 1085 | ||
1088 | ring->irq_put(ring); |
1086 | ring->irq_put(ring); |
1089 | trace_i915_gem_request_wait_end(ring, seqno); |
1087 | trace_i915_gem_request_wait_end(ring, seqno); |
1090 | #undef EXIT_COND |
1088 | #undef EXIT_COND |
1091 | 1089 | ||
1092 | if (timeout) { |
1090 | if (timeout) { |
1093 | // struct timespec sleep_time = timespec_sub(now, before); |
1091 | // struct timespec sleep_time = timespec_sub(now, before); |
1094 | // *timeout = timespec_sub(*timeout, sleep_time); |
1092 | // *timeout = timespec_sub(*timeout, sleep_time); |
1095 | } |
1093 | } |
1096 | 1094 | ||
1097 | switch (end) { |
1095 | switch (end) { |
1098 | case -EIO: |
1096 | case -EIO: |
1099 | case -EAGAIN: /* Wedged */ |
1097 | case -EAGAIN: /* Wedged */ |
1100 | case -ERESTARTSYS: /* Signal */ |
1098 | case -ERESTARTSYS: /* Signal */ |
1101 | return (int)end; |
1099 | return (int)end; |
1102 | case 0: /* Timeout */ |
1100 | case 0: /* Timeout */ |
1103 | if (timeout) |
1101 | if (timeout) |
1104 | set_normalized_timespec(timeout, 0, 0); |
1102 | set_normalized_timespec(timeout, 0, 0); |
1105 | return -ETIME; |
1103 | return -ETIME; |
1106 | default: /* Completed */ |
1104 | default: /* Completed */ |
1107 | WARN_ON(end < 0); /* We're not aware of other errors */ |
1105 | WARN_ON(end < 0); /* We're not aware of other errors */ |
1108 | return 0; |
1106 | return 0; |
1109 | } |
1107 | } |
1110 | } |
1108 | } |
1111 | 1109 | ||
1112 | /** |
1110 | /** |
1113 | * Waits for a sequence number to be signaled, and cleans up the |
1111 | * Waits for a sequence number to be signaled, and cleans up the |
1114 | * request and object lists appropriately for that event. |
1112 | * request and object lists appropriately for that event. |
1115 | */ |
1113 | */ |
1116 | int |
1114 | int |
1117 | i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) |
1115 | i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) |
1118 | { |
1116 | { |
1119 | struct drm_device *dev = ring->dev; |
1117 | struct drm_device *dev = ring->dev; |
1120 | struct drm_i915_private *dev_priv = dev->dev_private; |
1118 | struct drm_i915_private *dev_priv = dev->dev_private; |
1121 | bool interruptible = dev_priv->mm.interruptible; |
1119 | bool interruptible = dev_priv->mm.interruptible; |
1122 | int ret; |
1120 | int ret; |
1123 | 1121 | ||
1124 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
1122 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
1125 | BUG_ON(seqno == 0); |
1123 | BUG_ON(seqno == 0); |
1126 | 1124 | ||
1127 | ret = i915_gem_check_wedge(dev_priv, interruptible); |
1125 | ret = i915_gem_check_wedge(dev_priv, interruptible); |
1128 | if (ret) |
1126 | if (ret) |
1129 | return ret; |
1127 | return ret; |
1130 | 1128 | ||
1131 | ret = i915_gem_check_olr(ring, seqno); |
1129 | ret = i915_gem_check_olr(ring, seqno); |
1132 | if (ret) |
1130 | if (ret) |
1133 | return ret; |
1131 | return ret; |
1134 | 1132 | ||
1135 | return __wait_seqno(ring, seqno, interruptible, NULL); |
1133 | return __wait_seqno(ring, seqno, interruptible, NULL); |
1136 | } |
1134 | } |
1137 | 1135 | ||
1138 | /** |
1136 | /** |
1139 | * Ensures that all rendering to the object has completed and the object is |
1137 | * Ensures that all rendering to the object has completed and the object is |
1140 | * safe to unbind from the GTT or access from the CPU. |
1138 | * safe to unbind from the GTT or access from the CPU. |
1141 | */ |
1139 | */ |
1142 | static __must_check int |
1140 | static __must_check int |
1143 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, |
1141 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, |
1144 | bool readonly) |
1142 | bool readonly) |
1145 | { |
1143 | { |
1146 | struct intel_ring_buffer *ring = obj->ring; |
1144 | struct intel_ring_buffer *ring = obj->ring; |
1147 | u32 seqno; |
1145 | u32 seqno; |
1148 | int ret; |
1146 | int ret; |
1149 | 1147 | ||
1150 | seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; |
1148 | seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; |
1151 | if (seqno == 0) |
1149 | if (seqno == 0) |
1152 | return 0; |
1150 | return 0; |
1153 | 1151 | ||
1154 | ret = i915_wait_seqno(ring, seqno); |
1152 | ret = i915_wait_seqno(ring, seqno); |
1155 | if (ret) |
1153 | if (ret) |
1156 | return ret; |
1154 | return ret; |
1157 | 1155 | ||
1158 | i915_gem_retire_requests_ring(ring); |
1156 | i915_gem_retire_requests_ring(ring); |
1159 | 1157 | ||
1160 | /* Manually manage the write flush as we may have not yet |
1158 | /* Manually manage the write flush as we may have not yet |
1161 | * retired the buffer. |
1159 | * retired the buffer. |
1162 | */ |
1160 | */ |
1163 | if (obj->last_write_seqno && |
1161 | if (obj->last_write_seqno && |
1164 | i915_seqno_passed(seqno, obj->last_write_seqno)) { |
1162 | i915_seqno_passed(seqno, obj->last_write_seqno)) { |
1165 | obj->last_write_seqno = 0; |
1163 | obj->last_write_seqno = 0; |
1166 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
1164 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
1167 | } |
1165 | } |
1168 | 1166 | ||
1169 | return 0; |
1167 | return 0; |
1170 | } |
1168 | } |
1171 | 1169 | ||
1172 | /* A nonblocking variant of the above wait. This is a highly dangerous routine |
1170 | /* A nonblocking variant of the above wait. This is a highly dangerous routine |
1173 | * as the object state may change during this call. |
1171 | * as the object state may change during this call. |
1174 | */ |
1172 | */ |
1175 | static __must_check int |
1173 | static __must_check int |
1176 | i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, |
1174 | i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, |
1177 | bool readonly) |
1175 | bool readonly) |
1178 | { |
1176 | { |
1179 | struct drm_device *dev = obj->base.dev; |
1177 | struct drm_device *dev = obj->base.dev; |
1180 | struct drm_i915_private *dev_priv = dev->dev_private; |
1178 | struct drm_i915_private *dev_priv = dev->dev_private; |
1181 | struct intel_ring_buffer *ring = obj->ring; |
1179 | struct intel_ring_buffer *ring = obj->ring; |
1182 | u32 seqno; |
1180 | u32 seqno; |
1183 | int ret; |
1181 | int ret; |
1184 | 1182 | ||
1185 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
1183 | BUG_ON(!mutex_is_locked(&dev->struct_mutex)); |
1186 | BUG_ON(!dev_priv->mm.interruptible); |
1184 | BUG_ON(!dev_priv->mm.interruptible); |
1187 | 1185 | ||
1188 | seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; |
1186 | seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; |
1189 | if (seqno == 0) |
1187 | if (seqno == 0) |
1190 | return 0; |
1188 | return 0; |
1191 | 1189 | ||
1192 | ret = i915_gem_check_wedge(dev_priv, true); |
1190 | ret = i915_gem_check_wedge(dev_priv, true); |
1193 | if (ret) |
1191 | if (ret) |
1194 | return ret; |
1192 | return ret; |
1195 | 1193 | ||
1196 | ret = i915_gem_check_olr(ring, seqno); |
1194 | ret = i915_gem_check_olr(ring, seqno); |
1197 | if (ret) |
1195 | if (ret) |
1198 | return ret; |
1196 | return ret; |
1199 | 1197 | ||
1200 | mutex_unlock(&dev->struct_mutex); |
1198 | mutex_unlock(&dev->struct_mutex); |
1201 | ret = __wait_seqno(ring, seqno, true, NULL); |
1199 | ret = __wait_seqno(ring, seqno, true, NULL); |
1202 | mutex_lock(&dev->struct_mutex); |
1200 | mutex_lock(&dev->struct_mutex); |
1203 | 1201 | ||
1204 | i915_gem_retire_requests_ring(ring); |
1202 | i915_gem_retire_requests_ring(ring); |
1205 | 1203 | ||
1206 | /* Manually manage the write flush as we may have not yet |
1204 | /* Manually manage the write flush as we may have not yet |
1207 | * retired the buffer. |
1205 | * retired the buffer. |
1208 | */ |
1206 | */ |
1209 | if (obj->last_write_seqno && |
1207 | if (obj->last_write_seqno && |
1210 | i915_seqno_passed(seqno, obj->last_write_seqno)) { |
1208 | i915_seqno_passed(seqno, obj->last_write_seqno)) { |
1211 | obj->last_write_seqno = 0; |
1209 | obj->last_write_seqno = 0; |
1212 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
1210 | obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; |
1213 | } |
1211 | } |
1214 | 1212 | ||
1215 | return ret; |
1213 | return ret; |
1216 | } |
1214 | } |
1217 | 1215 | ||
1218 | /** |
1216 | /** |
1219 | * Called when user space prepares to use an object with the CPU, either |
1217 | * Called when user space prepares to use an object with the CPU, either |
1220 | * through the mmap ioctl's mapping or a GTT mapping. |
1218 | * through the mmap ioctl's mapping or a GTT mapping. |
1221 | */ |
1219 | */ |
1222 | int |
1220 | int |
1223 | i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, |
1221 | i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, |
1224 | struct drm_file *file) |
1222 | struct drm_file *file) |
1225 | { |
1223 | { |
1226 | struct drm_i915_gem_set_domain *args = data; |
1224 | struct drm_i915_gem_set_domain *args = data; |
1227 | struct drm_i915_gem_object *obj; |
1225 | struct drm_i915_gem_object *obj; |
1228 | uint32_t read_domains = args->read_domains; |
1226 | uint32_t read_domains = args->read_domains; |
1229 | uint32_t write_domain = args->write_domain; |
1227 | uint32_t write_domain = args->write_domain; |
1230 | int ret; |
1228 | int ret; |
1231 | 1229 | ||
1232 | /* Only handle setting domains to types used by the CPU. */ |
1230 | /* Only handle setting domains to types used by the CPU. */ |
1233 | if (write_domain & I915_GEM_GPU_DOMAINS) |
1231 | if (write_domain & I915_GEM_GPU_DOMAINS) |
1234 | return -EINVAL; |
1232 | return -EINVAL; |
1235 | 1233 | ||
1236 | if (read_domains & I915_GEM_GPU_DOMAINS) |
1234 | if (read_domains & I915_GEM_GPU_DOMAINS) |
1237 | return -EINVAL; |
1235 | return -EINVAL; |
1238 | 1236 | ||
1239 | /* Having something in the write domain implies it's in the read |
1237 | /* Having something in the write domain implies it's in the read |
1240 | * domain, and only that read domain. Enforce that in the request. |
1238 | * domain, and only that read domain. Enforce that in the request. |
1241 | */ |
1239 | */ |
1242 | if (write_domain != 0 && read_domains != write_domain) |
1240 | if (write_domain != 0 && read_domains != write_domain) |
1243 | return -EINVAL; |
1241 | return -EINVAL; |
1244 | 1242 | ||
1245 | ret = i915_mutex_lock_interruptible(dev); |
1243 | ret = i915_mutex_lock_interruptible(dev); |
1246 | if (ret) |
1244 | if (ret) |
1247 | return ret; |
1245 | return ret; |
1248 | 1246 | ||
1249 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
1247 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
1250 | if (&obj->base == NULL) { |
1248 | if (&obj->base == NULL) { |
1251 | ret = -ENOENT; |
1249 | ret = -ENOENT; |
1252 | goto unlock; |
1250 | goto unlock; |
1253 | } |
1251 | } |
1254 | 1252 | ||
1255 | /* Try to flush the object off the GPU without holding the lock. |
1253 | /* Try to flush the object off the GPU without holding the lock. |
1256 | * We will repeat the flush holding the lock in the normal manner |
1254 | * We will repeat the flush holding the lock in the normal manner |
1257 | * to catch cases where we are gazumped. |
1255 | * to catch cases where we are gazumped. |
1258 | */ |
1256 | */ |
1259 | ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain); |
1257 | ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain); |
1260 | if (ret) |
1258 | if (ret) |
1261 | goto unref; |
1259 | goto unref; |
1262 | 1260 | ||
1263 | if (read_domains & I915_GEM_DOMAIN_GTT) { |
1261 | if (read_domains & I915_GEM_DOMAIN_GTT) { |
1264 | ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); |
1262 | ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); |
1265 | 1263 | ||
1266 | /* Silently promote "you're not bound, there was nothing to do" |
1264 | /* Silently promote "you're not bound, there was nothing to do" |
1267 | * to success, since the client was just asking us to |
1265 | * to success, since the client was just asking us to |
1268 | * make sure everything was done. |
1266 | * make sure everything was done. |
1269 | */ |
1267 | */ |
1270 | if (ret == -EINVAL) |
1268 | if (ret == -EINVAL) |
1271 | ret = 0; |
1269 | ret = 0; |
1272 | } else { |
1270 | } else { |
1273 | ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); |
1271 | ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); |
1274 | } |
1272 | } |
1275 | 1273 | ||
1276 | unref: |
1274 | unref: |
1277 | drm_gem_object_unreference(&obj->base); |
1275 | drm_gem_object_unreference(&obj->base); |
1278 | unlock: |
1276 | unlock: |
1279 | mutex_unlock(&dev->struct_mutex); |
1277 | mutex_unlock(&dev->struct_mutex); |
1280 | return ret; |
1278 | return ret; |
1281 | } |
1279 | } |
1282 | 1280 | ||
1283 | 1281 | ||
1284 | 1282 | ||
1285 | 1283 | ||
1286 | 1284 | ||
1287 | 1285 | ||
1288 | /** |
1286 | /** |
1289 | * Maps the contents of an object, returning the address it is mapped |
1287 | * Maps the contents of an object, returning the address it is mapped |
1290 | * into. |
1288 | * into. |
1291 | * |
1289 | * |
1292 | * While the mapping holds a reference on the contents of the object, it doesn't |
1290 | * While the mapping holds a reference on the contents of the object, it doesn't |
1293 | * imply a ref on the object itself. |
1291 | * imply a ref on the object itself. |
1294 | */ |
1292 | */ |
1295 | int |
1293 | int |
1296 | i915_gem_mmap_ioctl(struct drm_device *dev, void *data, |
1294 | i915_gem_mmap_ioctl(struct drm_device *dev, void *data, |
1297 | struct drm_file *file) |
1295 | struct drm_file *file) |
1298 | { |
1296 | { |
1299 | struct drm_i915_gem_mmap *args = data; |
1297 | struct drm_i915_gem_mmap *args = data; |
1300 | struct drm_gem_object *obj; |
1298 | struct drm_gem_object *obj; |
1301 | unsigned long addr = 0; |
1299 | unsigned long addr = 0; |
1302 | 1300 | ||
1303 | obj = drm_gem_object_lookup(dev, file, args->handle); |
1301 | obj = drm_gem_object_lookup(dev, file, args->handle); |
1304 | if (obj == NULL) |
1302 | if (obj == NULL) |
1305 | return -ENOENT; |
1303 | return -ENOENT; |
1306 | 1304 | ||
1307 | dbgprintf("%s offset %lld size %lld\n", |
1305 | //dbgprintf("%s offset %lld size %lld\n", |
1308 | __FUNCTION__, args->offset, args->size); |
1306 | // __FUNCTION__, args->offset, args->size); |
1309 | /* prime objects have no backing filp to GEM mmap |
1307 | /* prime objects have no backing filp to GEM mmap |
1310 | * pages from. |
1308 | * pages from. |
1311 | */ |
1309 | */ |
1312 | if (!obj->filp) { |
1310 | if (!obj->filp) { |
1313 | drm_gem_object_unreference_unlocked(obj); |
1311 | drm_gem_object_unreference_unlocked(obj); |
1314 | return -EINVAL; |
1312 | return -EINVAL; |
1315 | } |
1313 | } |
1316 | 1314 | ||
1317 | addr = vm_mmap(obj->filp, 0, args->size, |
1315 | addr = vm_mmap(obj->filp, 0, args->size, |
1318 | PROT_READ | PROT_WRITE, MAP_SHARED, |
1316 | PROT_READ | PROT_WRITE, MAP_SHARED, |
1319 | args->offset); |
1317 | args->offset); |
1320 | drm_gem_object_unreference_unlocked(obj); |
1318 | drm_gem_object_unreference_unlocked(obj); |
1321 | if (IS_ERR((void *)addr)) |
1319 | if (IS_ERR((void *)addr)) |
1322 | return addr; |
1320 | return addr; |
1323 | 1321 | ||
1324 | args->addr_ptr = (uint64_t) addr; |
1322 | args->addr_ptr = (uint64_t) addr; |
1325 | 1323 | ||
1326 | return 0; |
1324 | return 0; |
1327 | } |
1325 | } |
1328 | 1326 | ||
1329 | 1327 | ||
1330 | 1328 | ||
1331 | 1329 | ||
1332 | 1330 | ||
1333 | 1331 | ||
1334 | 1332 | ||
1335 | 1333 | ||
1336 | 1334 | ||
1337 | 1335 | ||
1338 | 1336 | ||
1339 | 1337 | ||
1340 | 1338 | ||
1341 | /** |
1339 | /** |
1342 | * i915_gem_release_mmap - remove physical page mappings |
1340 | * i915_gem_release_mmap - remove physical page mappings |
1343 | * @obj: obj in question |
1341 | * @obj: obj in question |
1344 | * |
1342 | * |
1345 | * Preserve the reservation of the mmapping with the DRM core code, but |
1343 | * Preserve the reservation of the mmapping with the DRM core code, but |
1346 | * relinquish ownership of the pages back to the system. |
1344 | * relinquish ownership of the pages back to the system. |
1347 | * |
1345 | * |
1348 | * It is vital that we remove the page mapping if we have mapped a tiled |
1346 | * It is vital that we remove the page mapping if we have mapped a tiled |
1349 | * object through the GTT and then lose the fence register due to |
1347 | * object through the GTT and then lose the fence register due to |
1350 | * resource pressure. Similarly if the object has been moved out of the |
1348 | * resource pressure. Similarly if the object has been moved out of the |
1351 | * aperture, than pages mapped into userspace must be revoked. Removing the |
1349 | * aperture, than pages mapped into userspace must be revoked. Removing the |
1352 | * mapping will then trigger a page fault on the next user access, allowing |
1350 | * mapping will then trigger a page fault on the next user access, allowing |
1353 | * fixup by i915_gem_fault(). |
1351 | * fixup by i915_gem_fault(). |
1354 | */ |
1352 | */ |
1355 | void |
1353 | void |
1356 | i915_gem_release_mmap(struct drm_i915_gem_object *obj) |
1354 | i915_gem_release_mmap(struct drm_i915_gem_object *obj) |
1357 | { |
1355 | { |
1358 | if (!obj->fault_mappable) |
1356 | if (!obj->fault_mappable) |
1359 | return; |
1357 | return; |
1360 | 1358 | ||
1361 | if (obj->base.dev->dev_mapping) |
1359 | if (obj->base.dev->dev_mapping) |
1362 | // unmap_mapping_range(obj->base.dev->dev_mapping, |
1360 | // unmap_mapping_range(obj->base.dev->dev_mapping, |
1363 | // (loff_t)obj->base.map_list.hash.key< |
1361 | // (loff_t)obj->base.map_list.hash.key< |
1364 | // obj->base.size, 1); |
1362 | // obj->base.size, 1); |
1365 | 1363 | ||
1366 | obj->fault_mappable = false; |
1364 | obj->fault_mappable = false; |
1367 | } |
1365 | } |
1368 | 1366 | ||
1369 | static uint32_t |
1367 | static uint32_t |
1370 | i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) |
1368 | i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) |
1371 | { |
1369 | { |
1372 | uint32_t gtt_size; |
1370 | uint32_t gtt_size; |
1373 | 1371 | ||
1374 | if (INTEL_INFO(dev)->gen >= 4 || |
1372 | if (INTEL_INFO(dev)->gen >= 4 || |
1375 | tiling_mode == I915_TILING_NONE) |
1373 | tiling_mode == I915_TILING_NONE) |
1376 | return size; |
1374 | return size; |
1377 | 1375 | ||
1378 | /* Previous chips need a power-of-two fence region when tiling */ |
1376 | /* Previous chips need a power-of-two fence region when tiling */ |
1379 | if (INTEL_INFO(dev)->gen == 3) |
1377 | if (INTEL_INFO(dev)->gen == 3) |
1380 | gtt_size = 1024*1024; |
1378 | gtt_size = 1024*1024; |
1381 | else |
1379 | else |
1382 | gtt_size = 512*1024; |
1380 | gtt_size = 512*1024; |
1383 | 1381 | ||
1384 | while (gtt_size < size) |
1382 | while (gtt_size < size) |
1385 | gtt_size <<= 1; |
1383 | gtt_size <<= 1; |
1386 | 1384 | ||
1387 | return gtt_size; |
1385 | return gtt_size; |
1388 | } |
1386 | } |
1389 | 1387 | ||
1390 | /** |
1388 | /** |
1391 | * i915_gem_get_gtt_alignment - return required GTT alignment for an object |
1389 | * i915_gem_get_gtt_alignment - return required GTT alignment for an object |
1392 | * @obj: object to check |
1390 | * @obj: object to check |
1393 | * |
1391 | * |
1394 | * Return the required GTT alignment for an object, taking into account |
1392 | * Return the required GTT alignment for an object, taking into account |
1395 | * potential fence register mapping. |
1393 | * potential fence register mapping. |
1396 | */ |
1394 | */ |
1397 | static uint32_t |
1395 | static uint32_t |
1398 | i915_gem_get_gtt_alignment(struct drm_device *dev, |
1396 | i915_gem_get_gtt_alignment(struct drm_device *dev, |
1399 | uint32_t size, |
1397 | uint32_t size, |
1400 | int tiling_mode) |
1398 | int tiling_mode) |
1401 | { |
1399 | { |
1402 | /* |
1400 | /* |
1403 | * Minimum alignment is 4k (GTT page size), but might be greater |
1401 | * Minimum alignment is 4k (GTT page size), but might be greater |
1404 | * if a fence register is needed for the object. |
1402 | * if a fence register is needed for the object. |
1405 | */ |
1403 | */ |
1406 | if (INTEL_INFO(dev)->gen >= 4 || |
1404 | if (INTEL_INFO(dev)->gen >= 4 || |
1407 | tiling_mode == I915_TILING_NONE) |
1405 | tiling_mode == I915_TILING_NONE) |
1408 | return 4096; |
1406 | return 4096; |
1409 | 1407 | ||
1410 | /* |
1408 | /* |
1411 | * Previous chips need to be aligned to the size of the smallest |
1409 | * Previous chips need to be aligned to the size of the smallest |
1412 | * fence register that can contain the object. |
1410 | * fence register that can contain the object. |
1413 | */ |
1411 | */ |
1414 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
1412 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
1415 | } |
1413 | } |
1416 | 1414 | ||
1417 | /** |
1415 | /** |
1418 | * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an |
1416 | * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an |
1419 | * unfenced object |
1417 | * unfenced object |
1420 | * @dev: the device |
1418 | * @dev: the device |
1421 | * @size: size of the object |
1419 | * @size: size of the object |
1422 | * @tiling_mode: tiling mode of the object |
1420 | * @tiling_mode: tiling mode of the object |
1423 | * |
1421 | * |
1424 | * Return the required GTT alignment for an object, only taking into account |
1422 | * Return the required GTT alignment for an object, only taking into account |
1425 | * unfenced tiled surface requirements. |
1423 | * unfenced tiled surface requirements. |
1426 | */ |
1424 | */ |
1427 | uint32_t |
1425 | uint32_t |
1428 | i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, |
1426 | i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, |
1429 | uint32_t size, |
1427 | uint32_t size, |
1430 | int tiling_mode) |
1428 | int tiling_mode) |
1431 | { |
1429 | { |
1432 | /* |
1430 | /* |
1433 | * Minimum alignment is 4k (GTT page size) for sane hw. |
1431 | * Minimum alignment is 4k (GTT page size) for sane hw. |
1434 | */ |
1432 | */ |
1435 | if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || |
1433 | if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || |
1436 | tiling_mode == I915_TILING_NONE) |
1434 | tiling_mode == I915_TILING_NONE) |
1437 | return 4096; |
1435 | return 4096; |
1438 | 1436 | ||
1439 | /* Previous hardware however needs to be aligned to a power-of-two |
1437 | /* Previous hardware however needs to be aligned to a power-of-two |
1440 | * tile height. The simplest method for determining this is to reuse |
1438 | * tile height. The simplest method for determining this is to reuse |
1441 | * the power-of-tile object size. |
1439 | * the power-of-tile object size. |
1442 | */ |
1440 | */ |
1443 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
1441 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
1444 | } |
1442 | } |
1445 | 1443 | ||
1446 | /* Immediately discard the backing storage */ |
1444 | /* Immediately discard the backing storage */ |
1447 | static void |
1445 | static void |
1448 | i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
1446 | i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
1449 | { |
1447 | { |
1450 | // struct inode *inode; |
1448 | // struct inode *inode; |
1451 | 1449 | ||
1452 | // i915_gem_object_free_mmap_offset(obj); |
1450 | // i915_gem_object_free_mmap_offset(obj); |
1453 | 1451 | ||
1454 | if (obj->base.filp == NULL) |
1452 | if (obj->base.filp == NULL) |
1455 | return; |
1453 | return; |
1456 | 1454 | ||
1457 | /* Our goal here is to return as much of the memory as |
1455 | /* Our goal here is to return as much of the memory as |
1458 | * is possible back to the system as we are called from OOM. |
1456 | * is possible back to the system as we are called from OOM. |
1459 | * To do this we must instruct the shmfs to drop all of its |
1457 | * To do this we must instruct the shmfs to drop all of its |
1460 | * backing pages, *now*. |
1458 | * backing pages, *now*. |
1461 | */ |
1459 | */ |
1462 | // inode = obj->base.filp->f_path.dentry->d_inode; |
1460 | // inode = obj->base.filp->f_path.dentry->d_inode; |
1463 | // shmem_truncate_range(inode, 0, (loff_t)-1); |
1461 | // shmem_truncate_range(inode, 0, (loff_t)-1); |
1464 | 1462 | ||
1465 | obj->madv = __I915_MADV_PURGED; |
1463 | obj->madv = __I915_MADV_PURGED; |
1466 | } |
1464 | } |
1467 | 1465 | ||
1468 | static inline int |
1466 | static inline int |
1469 | i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) |
1467 | i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) |
1470 | { |
1468 | { |
1471 | return obj->madv == I915_MADV_DONTNEED; |
1469 | return obj->madv == I915_MADV_DONTNEED; |
1472 | } |
1470 | } |
1473 | 1471 | ||
1474 | static void |
1472 | static void |
1475 | i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) |
1473 | i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) |
1476 | { |
1474 | { |
1477 | int page_count = obj->base.size / PAGE_SIZE; |
1475 | int page_count = obj->base.size / PAGE_SIZE; |
1478 | struct scatterlist *sg; |
1476 | struct scatterlist *sg; |
1479 | int ret, i; |
1477 | int ret, i; |
1480 | 1478 | ||
1481 | BUG_ON(obj->madv == __I915_MADV_PURGED); |
1479 | BUG_ON(obj->madv == __I915_MADV_PURGED); |
1482 | 1480 | ||
1483 | ret = i915_gem_object_set_to_cpu_domain(obj, true); |
1481 | ret = i915_gem_object_set_to_cpu_domain(obj, true); |
1484 | if (ret) { |
1482 | if (ret) { |
1485 | /* In the event of a disaster, abandon all caches and |
1483 | /* In the event of a disaster, abandon all caches and |
1486 | * hope for the best. |
1484 | * hope for the best. |
1487 | */ |
1485 | */ |
1488 | WARN_ON(ret != -EIO); |
1486 | WARN_ON(ret != -EIO); |
1489 | i915_gem_clflush_object(obj); |
1487 | i915_gem_clflush_object(obj); |
1490 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1488 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1491 | } |
1489 | } |
1492 | 1490 | ||
1493 | if (obj->madv == I915_MADV_DONTNEED) |
1491 | if (obj->madv == I915_MADV_DONTNEED) |
1494 | obj->dirty = 0; |
1492 | obj->dirty = 0; |
1495 | 1493 | ||
1496 | for_each_sg(obj->pages->sgl, sg, page_count, i) { |
1494 | for_each_sg(obj->pages->sgl, sg, page_count, i) { |
1497 | struct page *page = sg_page(sg); |
1495 | struct page *page = sg_page(sg); |
1498 | 1496 | ||
1499 | page_cache_release(page); |
1497 | page_cache_release(page); |
1500 | } |
1498 | } |
1501 | //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count); |
1499 | //DRM_DEBUG_KMS("%s release %d pages\n", __FUNCTION__, page_count); |
1502 | printf("%s release %d pages\n", __FUNCTION__, page_count); |
- | |
1503 | 1500 | ||
1504 | obj->dirty = 0; |
1501 | obj->dirty = 0; |
1505 | 1502 | ||
1506 | sg_free_table(obj->pages); |
1503 | sg_free_table(obj->pages); |
1507 | kfree(obj->pages); |
1504 | kfree(obj->pages); |
1508 | } |
1505 | } |
1509 | 1506 | ||
1510 | static int |
1507 | static int |
1511 | i915_gem_object_put_pages(struct drm_i915_gem_object *obj) |
1508 | i915_gem_object_put_pages(struct drm_i915_gem_object *obj) |
1512 | { |
1509 | { |
1513 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
1510 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
1514 | 1511 | ||
1515 | if (obj->pages == NULL) |
1512 | if (obj->pages == NULL) |
1516 | return 0; |
1513 | return 0; |
1517 | 1514 | ||
1518 | BUG_ON(obj->gtt_space); |
1515 | BUG_ON(obj->gtt_space); |
1519 | 1516 | ||
1520 | if (obj->pages_pin_count) |
1517 | if (obj->pages_pin_count) |
1521 | return -EBUSY; |
1518 | return -EBUSY; |
1522 | 1519 | ||
1523 | /* ->put_pages might need to allocate memory for the bit17 swizzle |
1520 | /* ->put_pages might need to allocate memory for the bit17 swizzle |
1524 | * array, hence protect them from being reaped by removing them from gtt |
1521 | * array, hence protect them from being reaped by removing them from gtt |
1525 | * lists early. */ |
1522 | * lists early. */ |
1526 | list_del(&obj->gtt_list); |
1523 | list_del(&obj->gtt_list); |
1527 | 1524 | ||
1528 | ops->put_pages(obj); |
1525 | ops->put_pages(obj); |
1529 | obj->pages = NULL; |
1526 | obj->pages = NULL; |
1530 | 1527 | ||
1531 | if (i915_gem_object_is_purgeable(obj)) |
1528 | if (i915_gem_object_is_purgeable(obj)) |
1532 | i915_gem_object_truncate(obj); |
1529 | i915_gem_object_truncate(obj); |
1533 | 1530 | ||
1534 | return 0; |
1531 | return 0; |
1535 | } |
1532 | } |
1536 | 1533 | ||
1537 | 1534 | ||
1538 | 1535 | ||
1539 | 1536 | ||
1540 | 1537 | ||
1541 | 1538 | ||
1542 | 1539 | ||
1543 | 1540 | ||
1544 | static int |
1541 | static int |
1545 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) |
1542 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) |
1546 | { |
1543 | { |
1547 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
1544 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
1548 | int page_count, i; |
1545 | int page_count, i; |
1549 | struct address_space *mapping; |
1546 | struct address_space *mapping; |
1550 | struct sg_table *st; |
1547 | struct sg_table *st; |
1551 | struct scatterlist *sg; |
1548 | struct scatterlist *sg; |
1552 | struct page *page; |
1549 | struct page *page; |
1553 | gfp_t gfp; |
1550 | gfp_t gfp; |
1554 | 1551 | ||
1555 | /* Assert that the object is not currently in any GPU domain. As it |
1552 | /* Assert that the object is not currently in any GPU domain. As it |
1556 | * wasn't in the GTT, there shouldn't be any way it could have been in |
1553 | * wasn't in the GTT, there shouldn't be any way it could have been in |
1557 | * a GPU cache |
1554 | * a GPU cache |
1558 | */ |
1555 | */ |
1559 | BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); |
1556 | BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); |
1560 | BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); |
1557 | BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); |
1561 | 1558 | ||
1562 | st = kmalloc(sizeof(*st), GFP_KERNEL); |
1559 | st = kmalloc(sizeof(*st), GFP_KERNEL); |
1563 | if (st == NULL) |
1560 | if (st == NULL) |
1564 | return -ENOMEM; |
1561 | return -ENOMEM; |
1565 | 1562 | ||
1566 | page_count = obj->base.size / PAGE_SIZE; |
1563 | page_count = obj->base.size / PAGE_SIZE; |
1567 | if (sg_alloc_table(st, page_count, GFP_KERNEL)) { |
1564 | if (sg_alloc_table(st, page_count, GFP_KERNEL)) { |
1568 | sg_free_table(st); |
1565 | sg_free_table(st); |
1569 | kfree(st); |
1566 | kfree(st); |
1570 | return -ENOMEM; |
1567 | return -ENOMEM; |
1571 | } |
1568 | } |
1572 | 1569 | ||
1573 | /* Get the list of pages out of our struct file. They'll be pinned |
1570 | /* Get the list of pages out of our struct file. They'll be pinned |
1574 | * at this point until we release them. |
1571 | * at this point until we release them. |
1575 | * |
1572 | * |
1576 | * Fail silently without starting the shrinker |
1573 | * Fail silently without starting the shrinker |
1577 | */ |
1574 | */ |
1578 | for_each_sg(st->sgl, sg, page_count, i) { |
1575 | for_each_sg(st->sgl, sg, page_count, i) { |
1579 | page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp); |
1576 | page = shmem_read_mapping_page_gfp(obj->base.filp, i, gfp); |
1580 | if (IS_ERR(page)) { |
1577 | if (IS_ERR(page)) { |
1581 | dbgprintf("%s invalid page %p\n", __FUNCTION__, page); |
1578 | dbgprintf("%s invalid page %p\n", __FUNCTION__, page); |
1582 | goto err_pages; |
1579 | goto err_pages; |
1583 | 1580 | ||
1584 | } |
1581 | } |
1585 | sg_set_page(sg, page, PAGE_SIZE, 0); |
1582 | sg_set_page(sg, page, PAGE_SIZE, 0); |
1586 | } |
1583 | } |
1587 | 1584 | ||
1588 | obj->pages = st; |
1585 | obj->pages = st; |
1589 | 1586 | ||
1590 | DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count); |
1587 | // DRM_DEBUG_KMS("%s alloc %d pages\n", __FUNCTION__, page_count); |
1591 | 1588 | ||
1592 | return 0; |
1589 | return 0; |
1593 | 1590 | ||
1594 | err_pages: |
1591 | err_pages: |
1595 | for_each_sg(st->sgl, sg, i, page_count) |
1592 | for_each_sg(st->sgl, sg, i, page_count) |
1596 | page_cache_release(sg_page(sg)); |
1593 | page_cache_release(sg_page(sg)); |
1597 | sg_free_table(st); |
1594 | sg_free_table(st); |
1598 | kfree(st); |
1595 | kfree(st); |
1599 | return PTR_ERR(page); |
1596 | return PTR_ERR(page); |
1600 | } |
1597 | } |
1601 | 1598 | ||
1602 | /* Ensure that the associated pages are gathered from the backing storage |
1599 | /* Ensure that the associated pages are gathered from the backing storage |
1603 | * and pinned into our object. i915_gem_object_get_pages() may be called |
1600 | * and pinned into our object. i915_gem_object_get_pages() may be called |
1604 | * multiple times before they are released by a single call to |
1601 | * multiple times before they are released by a single call to |
1605 | * i915_gem_object_put_pages() - once the pages are no longer referenced |
1602 | * i915_gem_object_put_pages() - once the pages are no longer referenced |
1606 | * either as a result of memory pressure (reaping pages under the shrinker) |
1603 | * either as a result of memory pressure (reaping pages under the shrinker) |
1607 | * or as the object is itself released. |
1604 | * or as the object is itself released. |
1608 | */ |
1605 | */ |
1609 | int |
1606 | int |
1610 | i915_gem_object_get_pages(struct drm_i915_gem_object *obj) |
1607 | i915_gem_object_get_pages(struct drm_i915_gem_object *obj) |
1611 | { |
1608 | { |
1612 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
1609 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
1613 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
1610 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
1614 | int ret; |
1611 | int ret; |
1615 | 1612 | ||
1616 | if (obj->pages) |
1613 | if (obj->pages) |
1617 | return 0; |
1614 | return 0; |
1618 | 1615 | ||
1619 | BUG_ON(obj->pages_pin_count); |
1616 | BUG_ON(obj->pages_pin_count); |
1620 | 1617 | ||
1621 | ret = ops->get_pages(obj); |
1618 | ret = ops->get_pages(obj); |
1622 | if (ret) |
1619 | if (ret) |
1623 | return ret; |
1620 | return ret; |
1624 | 1621 | ||
1625 | list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
1622 | list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
1626 | return 0; |
1623 | return 0; |
1627 | } |
1624 | } |
1628 | 1625 | ||
1629 | void |
1626 | void |
1630 | i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, |
1627 | i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, |
1631 | struct intel_ring_buffer *ring) |
1628 | struct intel_ring_buffer *ring) |
1632 | { |
1629 | { |
1633 | struct drm_device *dev = obj->base.dev; |
1630 | struct drm_device *dev = obj->base.dev; |
1634 | struct drm_i915_private *dev_priv = dev->dev_private; |
1631 | struct drm_i915_private *dev_priv = dev->dev_private; |
1635 | u32 seqno = intel_ring_get_seqno(ring); |
1632 | u32 seqno = intel_ring_get_seqno(ring); |
1636 | 1633 | ||
1637 | BUG_ON(ring == NULL); |
1634 | BUG_ON(ring == NULL); |
1638 | obj->ring = ring; |
1635 | obj->ring = ring; |
1639 | 1636 | ||
1640 | /* Add a reference if we're newly entering the active list. */ |
1637 | /* Add a reference if we're newly entering the active list. */ |
1641 | if (!obj->active) { |
1638 | if (!obj->active) { |
1642 | drm_gem_object_reference(&obj->base); |
1639 | drm_gem_object_reference(&obj->base); |
1643 | obj->active = 1; |
1640 | obj->active = 1; |
1644 | } |
1641 | } |
1645 | 1642 | ||
1646 | /* Move from whatever list we were on to the tail of execution. */ |
1643 | /* Move from whatever list we were on to the tail of execution. */ |
1647 | list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); |
1644 | list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); |
1648 | list_move_tail(&obj->ring_list, &ring->active_list); |
1645 | list_move_tail(&obj->ring_list, &ring->active_list); |
1649 | 1646 | ||
1650 | obj->last_read_seqno = seqno; |
1647 | obj->last_read_seqno = seqno; |
1651 | 1648 | ||
1652 | if (obj->fenced_gpu_access) { |
1649 | if (obj->fenced_gpu_access) { |
1653 | obj->last_fenced_seqno = seqno; |
1650 | obj->last_fenced_seqno = seqno; |
1654 | 1651 | ||
1655 | /* Bump MRU to take account of the delayed flush */ |
1652 | /* Bump MRU to take account of the delayed flush */ |
1656 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
1653 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
1657 | struct drm_i915_fence_reg *reg; |
1654 | struct drm_i915_fence_reg *reg; |
1658 | 1655 | ||
1659 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
1656 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
1660 | list_move_tail(®->lru_list, |
1657 | list_move_tail(®->lru_list, |
1661 | &dev_priv->mm.fence_list); |
1658 | &dev_priv->mm.fence_list); |
1662 | } |
1659 | } |
1663 | } |
1660 | } |
1664 | } |
1661 | } |
1665 | 1662 | ||
1666 | static void |
1663 | static void |
1667 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
1664 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
1668 | { |
1665 | { |
1669 | struct drm_device *dev = obj->base.dev; |
1666 | struct drm_device *dev = obj->base.dev; |
1670 | struct drm_i915_private *dev_priv = dev->dev_private; |
1667 | struct drm_i915_private *dev_priv = dev->dev_private; |
1671 | 1668 | ||
1672 | BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); |
1669 | BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); |
1673 | BUG_ON(!obj->active); |
1670 | BUG_ON(!obj->active); |
1674 | 1671 | ||
1675 | if (obj->pin_count) /* are we a framebuffer? */ |
1672 | if (obj->pin_count) /* are we a framebuffer? */ |
1676 | intel_mark_fb_idle(obj); |
1673 | intel_mark_fb_idle(obj); |
1677 | 1674 | ||
1678 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
1675 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
1679 | 1676 | ||
1680 | list_del_init(&obj->ring_list); |
1677 | list_del_init(&obj->ring_list); |
1681 | obj->ring = NULL; |
1678 | obj->ring = NULL; |
1682 | 1679 | ||
1683 | obj->last_read_seqno = 0; |
1680 | obj->last_read_seqno = 0; |
1684 | obj->last_write_seqno = 0; |
1681 | obj->last_write_seqno = 0; |
1685 | obj->base.write_domain = 0; |
1682 | obj->base.write_domain = 0; |
1686 | 1683 | ||
1687 | obj->last_fenced_seqno = 0; |
1684 | obj->last_fenced_seqno = 0; |
1688 | obj->fenced_gpu_access = false; |
1685 | obj->fenced_gpu_access = false; |
1689 | 1686 | ||
1690 | obj->active = 0; |
1687 | obj->active = 0; |
1691 | drm_gem_object_unreference(&obj->base); |
1688 | drm_gem_object_unreference(&obj->base); |
1692 | 1689 | ||
1693 | WARN_ON(i915_verify_lists(dev)); |
1690 | WARN_ON(i915_verify_lists(dev)); |
1694 | } |
1691 | } |
1695 | 1692 | ||
1696 | static int |
1693 | static int |
1697 | i915_gem_handle_seqno_wrap(struct drm_device *dev) |
1694 | i915_gem_handle_seqno_wrap(struct drm_device *dev) |
1698 | { |
1695 | { |
1699 | struct drm_i915_private *dev_priv = dev->dev_private; |
1696 | struct drm_i915_private *dev_priv = dev->dev_private; |
1700 | struct intel_ring_buffer *ring; |
1697 | struct intel_ring_buffer *ring; |
1701 | int ret, i, j; |
1698 | int ret, i, j; |
1702 | 1699 | ||
1703 | /* The hardware uses various monotonic 32-bit counters, if we |
1700 | /* The hardware uses various monotonic 32-bit counters, if we |
1704 | * detect that they will wraparound we need to idle the GPU |
1701 | * detect that they will wraparound we need to idle the GPU |
1705 | * and reset those counters. |
1702 | * and reset those counters. |
1706 | */ |
1703 | */ |
1707 | ret = 0; |
1704 | ret = 0; |
1708 | for_each_ring(ring, dev_priv, i) { |
1705 | for_each_ring(ring, dev_priv, i) { |
1709 | for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) |
1706 | for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) |
1710 | ret |= ring->sync_seqno[j] != 0; |
1707 | ret |= ring->sync_seqno[j] != 0; |
1711 | } |
1708 | } |
1712 | if (ret == 0) |
1709 | if (ret == 0) |
1713 | return ret; |
1710 | return ret; |
1714 | 1711 | ||
1715 | ret = i915_gpu_idle(dev); |
1712 | ret = i915_gpu_idle(dev); |
1716 | if (ret) |
1713 | if (ret) |
1717 | return ret; |
1714 | return ret; |
1718 | 1715 | ||
1719 | i915_gem_retire_requests(dev); |
1716 | i915_gem_retire_requests(dev); |
1720 | for_each_ring(ring, dev_priv, i) { |
1717 | for_each_ring(ring, dev_priv, i) { |
1721 | for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) |
1718 | for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) |
1722 | ring->sync_seqno[j] = 0; |
1719 | ring->sync_seqno[j] = 0; |
1723 | } |
1720 | } |
1724 | 1721 | ||
1725 | return 0; |
1722 | return 0; |
1726 | } |
1723 | } |
1727 | 1724 | ||
1728 | int |
1725 | int |
1729 | i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) |
1726 | i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) |
1730 | { |
1727 | { |
1731 | struct drm_i915_private *dev_priv = dev->dev_private; |
1728 | struct drm_i915_private *dev_priv = dev->dev_private; |
1732 | 1729 | ||
1733 | /* reserve 0 for non-seqno */ |
1730 | /* reserve 0 for non-seqno */ |
1734 | if (dev_priv->next_seqno == 0) { |
1731 | if (dev_priv->next_seqno == 0) { |
1735 | int ret = i915_gem_handle_seqno_wrap(dev); |
1732 | int ret = i915_gem_handle_seqno_wrap(dev); |
1736 | if (ret) |
1733 | if (ret) |
1737 | return ret; |
1734 | return ret; |
1738 | 1735 | ||
1739 | dev_priv->next_seqno = 1; |
1736 | dev_priv->next_seqno = 1; |
1740 | } |
1737 | } |
1741 | 1738 | ||
1742 | *seqno = dev_priv->next_seqno++; |
1739 | *seqno = dev_priv->next_seqno++; |
1743 | return 0; |
1740 | return 0; |
1744 | } |
1741 | } |
1745 | 1742 | ||
1746 | int |
1743 | int |
1747 | i915_add_request(struct intel_ring_buffer *ring, |
1744 | i915_add_request(struct intel_ring_buffer *ring, |
1748 | struct drm_file *file, |
1745 | struct drm_file *file, |
1749 | u32 *out_seqno) |
1746 | u32 *out_seqno) |
1750 | { |
1747 | { |
1751 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1748 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1752 | struct drm_i915_gem_request *request; |
1749 | struct drm_i915_gem_request *request; |
1753 | u32 request_ring_position; |
1750 | u32 request_ring_position; |
1754 | int was_empty; |
1751 | int was_empty; |
1755 | int ret; |
1752 | int ret; |
1756 | 1753 | ||
1757 | /* |
1754 | /* |
1758 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
1755 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
1759 | * after having emitted the batchbuffer command. Hence we need to fix |
1756 | * after having emitted the batchbuffer command. Hence we need to fix |
1760 | * things up similar to emitting the lazy request. The difference here |
1757 | * things up similar to emitting the lazy request. The difference here |
1761 | * is that the flush _must_ happen before the next request, no matter |
1758 | * is that the flush _must_ happen before the next request, no matter |
1762 | * what. |
1759 | * what. |
1763 | */ |
1760 | */ |
1764 | ret = intel_ring_flush_all_caches(ring); |
1761 | ret = intel_ring_flush_all_caches(ring); |
1765 | if (ret) |
1762 | if (ret) |
1766 | return ret; |
1763 | return ret; |
1767 | 1764 | ||
1768 | request = kmalloc(sizeof(*request), GFP_KERNEL); |
1765 | request = kmalloc(sizeof(*request), GFP_KERNEL); |
1769 | if (request == NULL) |
1766 | if (request == NULL) |
1770 | return -ENOMEM; |
1767 | return -ENOMEM; |
1771 | 1768 | ||
1772 | 1769 | ||
1773 | /* Record the position of the start of the request so that |
1770 | /* Record the position of the start of the request so that |
1774 | * should we detect the updated seqno part-way through the |
1771 | * should we detect the updated seqno part-way through the |
1775 | * GPU processing the request, we never over-estimate the |
1772 | * GPU processing the request, we never over-estimate the |
1776 | * position of the head. |
1773 | * position of the head. |
1777 | */ |
1774 | */ |
1778 | request_ring_position = intel_ring_get_tail(ring); |
1775 | request_ring_position = intel_ring_get_tail(ring); |
1779 | 1776 | ||
1780 | ret = ring->add_request(ring); |
1777 | ret = ring->add_request(ring); |
1781 | if (ret) { |
1778 | if (ret) { |
1782 | kfree(request); |
1779 | kfree(request); |
1783 | return ret; |
1780 | return ret; |
1784 | } |
1781 | } |
1785 | 1782 | ||
1786 | request->seqno = intel_ring_get_seqno(ring); |
1783 | request->seqno = intel_ring_get_seqno(ring); |
1787 | request->ring = ring; |
1784 | request->ring = ring; |
1788 | request->tail = request_ring_position; |
1785 | request->tail = request_ring_position; |
1789 | request->emitted_jiffies = GetTimerTicks(); |
1786 | request->emitted_jiffies = GetTimerTicks(); |
1790 | was_empty = list_empty(&ring->request_list); |
1787 | was_empty = list_empty(&ring->request_list); |
1791 | list_add_tail(&request->list, &ring->request_list); |
1788 | list_add_tail(&request->list, &ring->request_list); |
1792 | request->file_priv = NULL; |
1789 | request->file_priv = NULL; |
1793 | 1790 | ||
1794 | if (file) { |
1791 | if (file) { |
1795 | struct drm_i915_file_private *file_priv = file->driver_priv; |
1792 | struct drm_i915_file_private *file_priv = file->driver_priv; |
1796 | 1793 | ||
1797 | spin_lock(&file_priv->mm.lock); |
1794 | spin_lock(&file_priv->mm.lock); |
1798 | request->file_priv = file_priv; |
1795 | request->file_priv = file_priv; |
1799 | list_add_tail(&request->client_list, |
1796 | list_add_tail(&request->client_list, |
1800 | &file_priv->mm.request_list); |
1797 | &file_priv->mm.request_list); |
1801 | spin_unlock(&file_priv->mm.lock); |
1798 | spin_unlock(&file_priv->mm.lock); |
1802 | } |
1799 | } |
1803 | 1800 | ||
1804 | trace_i915_gem_request_add(ring, request->seqno); |
1801 | trace_i915_gem_request_add(ring, request->seqno); |
1805 | ring->outstanding_lazy_request = 0; |
1802 | ring->outstanding_lazy_request = 0; |
1806 | 1803 | ||
1807 | if (!dev_priv->mm.suspended) { |
1804 | if (!dev_priv->mm.suspended) { |
1808 | if (i915_enable_hangcheck) { |
1805 | if (i915_enable_hangcheck) { |
1809 | // mod_timer(&dev_priv->hangcheck_timer, |
1806 | // mod_timer(&dev_priv->hangcheck_timer, |
1810 | // jiffies + |
1807 | // jiffies + |
1811 | // msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); |
1808 | // msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); |
1812 | } |
1809 | } |
1813 | if (was_empty) { |
1810 | if (was_empty) { |
1814 | queue_delayed_work(dev_priv->wq, |
1811 | queue_delayed_work(dev_priv->wq, |
1815 | &dev_priv->mm.retire_work, HZ); |
1812 | &dev_priv->mm.retire_work, HZ); |
1816 | intel_mark_busy(dev_priv->dev); |
1813 | intel_mark_busy(dev_priv->dev); |
1817 | } |
1814 | } |
1818 | } |
1815 | } |
1819 | 1816 | ||
1820 | if (out_seqno) |
1817 | if (out_seqno) |
1821 | *out_seqno = request->seqno; |
1818 | *out_seqno = request->seqno; |
1822 | return 0; |
1819 | return 0; |
1823 | } |
1820 | } |
1824 | 1821 | ||
1825 | static inline void |
1822 | static inline void |
1826 | i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) |
1823 | i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) |
1827 | { |
1824 | { |
1828 | struct drm_i915_file_private *file_priv = request->file_priv; |
1825 | struct drm_i915_file_private *file_priv = request->file_priv; |
1829 | 1826 | ||
1830 | if (!file_priv) |
1827 | if (!file_priv) |
1831 | return; |
1828 | return; |
1832 | 1829 | ||
1833 | spin_lock(&file_priv->mm.lock); |
1830 | spin_lock(&file_priv->mm.lock); |
1834 | if (request->file_priv) { |
1831 | if (request->file_priv) { |
1835 | list_del(&request->client_list); |
1832 | list_del(&request->client_list); |
1836 | request->file_priv = NULL; |
1833 | request->file_priv = NULL; |
1837 | } |
1834 | } |
1838 | spin_unlock(&file_priv->mm.lock); |
1835 | spin_unlock(&file_priv->mm.lock); |
1839 | } |
1836 | } |
1840 | 1837 | ||
1841 | static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, |
1838 | static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, |
1842 | struct intel_ring_buffer *ring) |
1839 | struct intel_ring_buffer *ring) |
1843 | { |
1840 | { |
1844 | while (!list_empty(&ring->request_list)) { |
1841 | while (!list_empty(&ring->request_list)) { |
1845 | struct drm_i915_gem_request *request; |
1842 | struct drm_i915_gem_request *request; |
1846 | 1843 | ||
1847 | request = list_first_entry(&ring->request_list, |
1844 | request = list_first_entry(&ring->request_list, |
1848 | struct drm_i915_gem_request, |
1845 | struct drm_i915_gem_request, |
1849 | list); |
1846 | list); |
1850 | 1847 | ||
1851 | list_del(&request->list); |
1848 | list_del(&request->list); |
1852 | i915_gem_request_remove_from_client(request); |
1849 | i915_gem_request_remove_from_client(request); |
1853 | kfree(request); |
1850 | kfree(request); |
1854 | } |
1851 | } |
1855 | 1852 | ||
1856 | while (!list_empty(&ring->active_list)) { |
1853 | while (!list_empty(&ring->active_list)) { |
1857 | struct drm_i915_gem_object *obj; |
1854 | struct drm_i915_gem_object *obj; |
1858 | 1855 | ||
1859 | obj = list_first_entry(&ring->active_list, |
1856 | obj = list_first_entry(&ring->active_list, |
1860 | struct drm_i915_gem_object, |
1857 | struct drm_i915_gem_object, |
1861 | ring_list); |
1858 | ring_list); |
1862 | 1859 | ||
1863 | i915_gem_object_move_to_inactive(obj); |
1860 | i915_gem_object_move_to_inactive(obj); |
1864 | } |
1861 | } |
1865 | } |
1862 | } |
1866 | 1863 | ||
1867 | static void i915_gem_reset_fences(struct drm_device *dev) |
1864 | static void i915_gem_reset_fences(struct drm_device *dev) |
1868 | { |
1865 | { |
1869 | struct drm_i915_private *dev_priv = dev->dev_private; |
1866 | struct drm_i915_private *dev_priv = dev->dev_private; |
1870 | int i; |
1867 | int i; |
1871 | 1868 | ||
1872 | for (i = 0; i < dev_priv->num_fence_regs; i++) { |
1869 | for (i = 0; i < dev_priv->num_fence_regs; i++) { |
1873 | struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
1870 | struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
1874 | 1871 | ||
1875 | i915_gem_write_fence(dev, i, NULL); |
1872 | i915_gem_write_fence(dev, i, NULL); |
1876 | 1873 | ||
1877 | if (reg->obj) |
1874 | if (reg->obj) |
1878 | i915_gem_object_fence_lost(reg->obj); |
1875 | i915_gem_object_fence_lost(reg->obj); |
1879 | 1876 | ||
1880 | reg->pin_count = 0; |
1877 | reg->pin_count = 0; |
1881 | reg->obj = NULL; |
1878 | reg->obj = NULL; |
1882 | INIT_LIST_HEAD(®->lru_list); |
1879 | INIT_LIST_HEAD(®->lru_list); |
1883 | } |
1880 | } |
1884 | 1881 | ||
1885 | INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
1882 | INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
1886 | } |
1883 | } |
1887 | 1884 | ||
1888 | void i915_gem_reset(struct drm_device *dev) |
1885 | void i915_gem_reset(struct drm_device *dev) |
1889 | { |
1886 | { |
1890 | struct drm_i915_private *dev_priv = dev->dev_private; |
1887 | struct drm_i915_private *dev_priv = dev->dev_private; |
1891 | struct drm_i915_gem_object *obj; |
1888 | struct drm_i915_gem_object *obj; |
1892 | struct intel_ring_buffer *ring; |
1889 | struct intel_ring_buffer *ring; |
1893 | int i; |
1890 | int i; |
1894 | 1891 | ||
1895 | for_each_ring(ring, dev_priv, i) |
1892 | for_each_ring(ring, dev_priv, i) |
1896 | i915_gem_reset_ring_lists(dev_priv, ring); |
1893 | i915_gem_reset_ring_lists(dev_priv, ring); |
1897 | 1894 | ||
1898 | /* Move everything out of the GPU domains to ensure we do any |
1895 | /* Move everything out of the GPU domains to ensure we do any |
1899 | * necessary invalidation upon reuse. |
1896 | * necessary invalidation upon reuse. |
1900 | */ |
1897 | */ |
1901 | list_for_each_entry(obj, |
1898 | list_for_each_entry(obj, |
1902 | &dev_priv->mm.inactive_list, |
1899 | &dev_priv->mm.inactive_list, |
1903 | mm_list) |
1900 | mm_list) |
1904 | { |
1901 | { |
1905 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
1902 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
1906 | } |
1903 | } |
1907 | 1904 | ||
1908 | /* The fence registers are invalidated so clear them out */ |
1905 | /* The fence registers are invalidated so clear them out */ |
1909 | i915_gem_reset_fences(dev); |
1906 | i915_gem_reset_fences(dev); |
1910 | } |
1907 | } |
1911 | 1908 | ||
1912 | /** |
1909 | /** |
1913 | * This function clears the request list as sequence numbers are passed. |
1910 | * This function clears the request list as sequence numbers are passed. |
1914 | */ |
1911 | */ |
1915 | void |
1912 | void |
1916 | i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) |
1913 | i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) |
1917 | { |
1914 | { |
1918 | uint32_t seqno; |
1915 | uint32_t seqno; |
1919 | 1916 | ||
1920 | if (list_empty(&ring->request_list)) |
1917 | if (list_empty(&ring->request_list)) |
1921 | return; |
1918 | return; |
1922 | 1919 | ||
1923 | WARN_ON(i915_verify_lists(ring->dev)); |
1920 | WARN_ON(i915_verify_lists(ring->dev)); |
1924 | 1921 | ||
1925 | seqno = ring->get_seqno(ring, true); |
1922 | seqno = ring->get_seqno(ring, true); |
1926 | 1923 | ||
1927 | while (!list_empty(&ring->request_list)) { |
1924 | while (!list_empty(&ring->request_list)) { |
1928 | struct drm_i915_gem_request *request; |
1925 | struct drm_i915_gem_request *request; |
1929 | 1926 | ||
1930 | request = list_first_entry(&ring->request_list, |
1927 | request = list_first_entry(&ring->request_list, |
1931 | struct drm_i915_gem_request, |
1928 | struct drm_i915_gem_request, |
1932 | list); |
1929 | list); |
1933 | 1930 | ||
1934 | if (!i915_seqno_passed(seqno, request->seqno)) |
1931 | if (!i915_seqno_passed(seqno, request->seqno)) |
1935 | break; |
1932 | break; |
1936 | 1933 | ||
1937 | trace_i915_gem_request_retire(ring, request->seqno); |
1934 | trace_i915_gem_request_retire(ring, request->seqno); |
1938 | /* We know the GPU must have read the request to have |
1935 | /* We know the GPU must have read the request to have |
1939 | * sent us the seqno + interrupt, so use the position |
1936 | * sent us the seqno + interrupt, so use the position |
1940 | * of tail of the request to update the last known position |
1937 | * of tail of the request to update the last known position |
1941 | * of the GPU head. |
1938 | * of the GPU head. |
1942 | */ |
1939 | */ |
1943 | ring->last_retired_head = request->tail; |
1940 | ring->last_retired_head = request->tail; |
1944 | 1941 | ||
1945 | list_del(&request->list); |
1942 | list_del(&request->list); |
1946 | i915_gem_request_remove_from_client(request); |
1943 | i915_gem_request_remove_from_client(request); |
1947 | kfree(request); |
1944 | kfree(request); |
1948 | } |
1945 | } |
1949 | 1946 | ||
1950 | /* Move any buffers on the active list that are no longer referenced |
1947 | /* Move any buffers on the active list that are no longer referenced |
1951 | * by the ringbuffer to the flushing/inactive lists as appropriate. |
1948 | * by the ringbuffer to the flushing/inactive lists as appropriate. |
1952 | */ |
1949 | */ |
1953 | while (!list_empty(&ring->active_list)) { |
1950 | while (!list_empty(&ring->active_list)) { |
1954 | struct drm_i915_gem_object *obj; |
1951 | struct drm_i915_gem_object *obj; |
1955 | 1952 | ||
1956 | obj = list_first_entry(&ring->active_list, |
1953 | obj = list_first_entry(&ring->active_list, |
1957 | struct drm_i915_gem_object, |
1954 | struct drm_i915_gem_object, |
1958 | ring_list); |
1955 | ring_list); |
1959 | 1956 | ||
1960 | if (!i915_seqno_passed(seqno, obj->last_read_seqno)) |
1957 | if (!i915_seqno_passed(seqno, obj->last_read_seqno)) |
1961 | break; |
1958 | break; |
1962 | 1959 | ||
1963 | i915_gem_object_move_to_inactive(obj); |
1960 | i915_gem_object_move_to_inactive(obj); |
1964 | } |
1961 | } |
1965 | 1962 | ||
1966 | if (unlikely(ring->trace_irq_seqno && |
1963 | if (unlikely(ring->trace_irq_seqno && |
1967 | i915_seqno_passed(seqno, ring->trace_irq_seqno))) { |
1964 | i915_seqno_passed(seqno, ring->trace_irq_seqno))) { |
1968 | ring->irq_put(ring); |
1965 | ring->irq_put(ring); |
1969 | ring->trace_irq_seqno = 0; |
1966 | ring->trace_irq_seqno = 0; |
1970 | } |
1967 | } |
1971 | 1968 | ||
1972 | WARN_ON(i915_verify_lists(ring->dev)); |
1969 | WARN_ON(i915_verify_lists(ring->dev)); |
1973 | } |
1970 | } |
1974 | 1971 | ||
1975 | void |
1972 | void |
1976 | i915_gem_retire_requests(struct drm_device *dev) |
1973 | i915_gem_retire_requests(struct drm_device *dev) |
1977 | { |
1974 | { |
1978 | drm_i915_private_t *dev_priv = dev->dev_private; |
1975 | drm_i915_private_t *dev_priv = dev->dev_private; |
1979 | struct intel_ring_buffer *ring; |
1976 | struct intel_ring_buffer *ring; |
1980 | int i; |
1977 | int i; |
1981 | 1978 | ||
1982 | for_each_ring(ring, dev_priv, i) |
1979 | for_each_ring(ring, dev_priv, i) |
1983 | i915_gem_retire_requests_ring(ring); |
1980 | i915_gem_retire_requests_ring(ring); |
1984 | } |
1981 | } |
1985 | 1982 | ||
1986 | static void |
1983 | static void |
1987 | i915_gem_retire_work_handler(struct work_struct *work) |
1984 | i915_gem_retire_work_handler(struct work_struct *work) |
1988 | { |
1985 | { |
1989 | drm_i915_private_t *dev_priv; |
1986 | drm_i915_private_t *dev_priv; |
1990 | struct drm_device *dev; |
1987 | struct drm_device *dev; |
1991 | struct intel_ring_buffer *ring; |
1988 | struct intel_ring_buffer *ring; |
1992 | bool idle; |
1989 | bool idle; |
1993 | int i; |
1990 | int i; |
1994 | 1991 | ||
1995 | dev_priv = container_of(work, drm_i915_private_t, |
1992 | dev_priv = container_of(work, drm_i915_private_t, |
1996 | mm.retire_work.work); |
1993 | mm.retire_work.work); |
1997 | dev = dev_priv->dev; |
1994 | dev = dev_priv->dev; |
1998 | 1995 | ||
1999 | /* Come back later if the device is busy... */ |
1996 | /* Come back later if the device is busy... */ |
2000 | if (!mutex_trylock(&dev->struct_mutex)) { |
1997 | if (!mutex_trylock(&dev->struct_mutex)) { |
2001 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); |
1998 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); |
2002 | return; |
1999 | return; |
2003 | } |
2000 | } |
2004 | 2001 | ||
2005 | i915_gem_retire_requests(dev); |
2002 | i915_gem_retire_requests(dev); |
2006 | 2003 | ||
2007 | /* Send a periodic flush down the ring so we don't hold onto GEM |
2004 | /* Send a periodic flush down the ring so we don't hold onto GEM |
2008 | * objects indefinitely. |
2005 | * objects indefinitely. |
2009 | */ |
2006 | */ |
2010 | idle = true; |
2007 | idle = true; |
2011 | for_each_ring(ring, dev_priv, i) { |
2008 | for_each_ring(ring, dev_priv, i) { |
2012 | if (ring->gpu_caches_dirty) |
2009 | if (ring->gpu_caches_dirty) |
2013 | i915_add_request(ring, NULL, NULL); |
2010 | i915_add_request(ring, NULL, NULL); |
2014 | 2011 | ||
2015 | idle &= list_empty(&ring->request_list); |
2012 | idle &= list_empty(&ring->request_list); |
2016 | } |
2013 | } |
2017 | 2014 | ||
2018 | if (!dev_priv->mm.suspended && !idle) |
2015 | if (!dev_priv->mm.suspended && !idle) |
2019 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); |
2016 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); |
2020 | if (idle) |
2017 | if (idle) |
2021 | intel_mark_idle(dev); |
2018 | intel_mark_idle(dev); |
2022 | 2019 | ||
2023 | mutex_unlock(&dev->struct_mutex); |
2020 | mutex_unlock(&dev->struct_mutex); |
2024 | } |
2021 | } |
2025 | 2022 | ||
2026 | /** |
2023 | /** |
2027 | * Ensures that an object will eventually get non-busy by flushing any required |
2024 | * Ensures that an object will eventually get non-busy by flushing any required |
2028 | * write domains, emitting any outstanding lazy request and retiring and |
2025 | * write domains, emitting any outstanding lazy request and retiring and |
2029 | * completed requests. |
2026 | * completed requests. |
2030 | */ |
2027 | */ |
2031 | static int |
2028 | static int |
2032 | i915_gem_object_flush_active(struct drm_i915_gem_object *obj) |
2029 | i915_gem_object_flush_active(struct drm_i915_gem_object *obj) |
2033 | { |
2030 | { |
2034 | int ret; |
2031 | int ret; |
2035 | 2032 | ||
2036 | if (obj->active) { |
2033 | if (obj->active) { |
2037 | ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); |
2034 | ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); |
2038 | if (ret) |
2035 | if (ret) |
2039 | return ret; |
2036 | return ret; |
2040 | 2037 | ||
2041 | i915_gem_retire_requests_ring(obj->ring); |
2038 | i915_gem_retire_requests_ring(obj->ring); |
2042 | } |
2039 | } |
2043 | 2040 | ||
2044 | return 0; |
2041 | return 0; |
2045 | } |
2042 | } |
2046 | 2043 | ||
2047 | /** |
2044 | /** |
2048 | * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT |
2045 | * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT |
2049 | * @DRM_IOCTL_ARGS: standard ioctl arguments |
2046 | * @DRM_IOCTL_ARGS: standard ioctl arguments |
2050 | * |
2047 | * |
2051 | * Returns 0 if successful, else an error is returned with the remaining time in |
2048 | * Returns 0 if successful, else an error is returned with the remaining time in |
2052 | * the timeout parameter. |
2049 | * the timeout parameter. |
2053 | * -ETIME: object is still busy after timeout |
2050 | * -ETIME: object is still busy after timeout |
2054 | * -ERESTARTSYS: signal interrupted the wait |
2051 | * -ERESTARTSYS: signal interrupted the wait |
2055 | * -ENONENT: object doesn't exist |
2052 | * -ENONENT: object doesn't exist |
2056 | * Also possible, but rare: |
2053 | * Also possible, but rare: |
2057 | * -EAGAIN: GPU wedged |
2054 | * -EAGAIN: GPU wedged |
2058 | * -ENOMEM: damn |
2055 | * -ENOMEM: damn |
2059 | * -ENODEV: Internal IRQ fail |
2056 | * -ENODEV: Internal IRQ fail |
2060 | * -E?: The add request failed |
2057 | * -E?: The add request failed |
2061 | * |
2058 | * |
2062 | * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any |
2059 | * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any |
2063 | * non-zero timeout parameter the wait ioctl will wait for the given number of |
2060 | * non-zero timeout parameter the wait ioctl will wait for the given number of |
2064 | * nanoseconds on an object becoming unbusy. Since the wait itself does so |
2061 | * nanoseconds on an object becoming unbusy. Since the wait itself does so |
2065 | * without holding struct_mutex the object may become re-busied before this |
2062 | * without holding struct_mutex the object may become re-busied before this |
2066 | * function completes. A similar but shorter * race condition exists in the busy |
2063 | * function completes. A similar but shorter * race condition exists in the busy |
2067 | * ioctl |
2064 | * ioctl |
2068 | */ |
2065 | */ |
2069 | 2066 | ||
2070 | 2067 | ||
2071 | 2068 | ||
2072 | 2069 | ||
2073 | 2070 | ||
2074 | 2071 | ||
2075 | 2072 | ||
2076 | 2073 | ||
2077 | 2074 | ||
2078 | 2075 | ||
2079 | 2076 | ||
2080 | 2077 | ||
2081 | /** |
2078 | /** |
2082 | * i915_gem_object_sync - sync an object to a ring. |
2079 | * i915_gem_object_sync - sync an object to a ring. |
2083 | * |
2080 | * |
2084 | * @obj: object which may be in use on another ring. |
2081 | * @obj: object which may be in use on another ring. |
2085 | * @to: ring we wish to use the object on. May be NULL. |
2082 | * @to: ring we wish to use the object on. May be NULL. |
2086 | * |
2083 | * |
2087 | * This code is meant to abstract object synchronization with the GPU. |
2084 | * This code is meant to abstract object synchronization with the GPU. |
2088 | * Calling with NULL implies synchronizing the object with the CPU |
2085 | * Calling with NULL implies synchronizing the object with the CPU |
2089 | * rather than a particular GPU ring. |
2086 | * rather than a particular GPU ring. |
2090 | * |
2087 | * |
2091 | * Returns 0 if successful, else propagates up the lower layer error. |
2088 | * Returns 0 if successful, else propagates up the lower layer error. |
2092 | */ |
2089 | */ |
2093 | int |
2090 | int |
2094 | i915_gem_object_sync(struct drm_i915_gem_object *obj, |
2091 | i915_gem_object_sync(struct drm_i915_gem_object *obj, |
2095 | struct intel_ring_buffer *to) |
2092 | struct intel_ring_buffer *to) |
2096 | { |
2093 | { |
2097 | struct intel_ring_buffer *from = obj->ring; |
2094 | struct intel_ring_buffer *from = obj->ring; |
2098 | u32 seqno; |
2095 | u32 seqno; |
2099 | int ret, idx; |
2096 | int ret, idx; |
2100 | 2097 | ||
2101 | if (from == NULL || to == from) |
2098 | if (from == NULL || to == from) |
2102 | return 0; |
2099 | return 0; |
2103 | 2100 | ||
2104 | if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) |
2101 | if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) |
2105 | return i915_gem_object_wait_rendering(obj, false); |
2102 | return i915_gem_object_wait_rendering(obj, false); |
2106 | 2103 | ||
2107 | idx = intel_ring_sync_index(from, to); |
2104 | idx = intel_ring_sync_index(from, to); |
2108 | 2105 | ||
2109 | seqno = obj->last_read_seqno; |
2106 | seqno = obj->last_read_seqno; |
2110 | if (seqno <= from->sync_seqno[idx]) |
2107 | if (seqno <= from->sync_seqno[idx]) |
2111 | return 0; |
2108 | return 0; |
2112 | 2109 | ||
2113 | ret = i915_gem_check_olr(obj->ring, seqno); |
2110 | ret = i915_gem_check_olr(obj->ring, seqno); |
2114 | if (ret) |
2111 | if (ret) |
2115 | return ret; |
2112 | return ret; |
2116 | 2113 | ||
2117 | ret = to->sync_to(to, from, seqno); |
2114 | ret = to->sync_to(to, from, seqno); |
2118 | if (!ret) |
2115 | if (!ret) |
2119 | /* We use last_read_seqno because sync_to() |
2116 | /* We use last_read_seqno because sync_to() |
2120 | * might have just caused seqno wrap under |
2117 | * might have just caused seqno wrap under |
2121 | * the radar. |
2118 | * the radar. |
2122 | */ |
2119 | */ |
2123 | from->sync_seqno[idx] = obj->last_read_seqno; |
2120 | from->sync_seqno[idx] = obj->last_read_seqno; |
2124 | 2121 | ||
2125 | return ret; |
2122 | return ret; |
2126 | } |
2123 | } |
2127 | 2124 | ||
2128 | static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) |
2125 | static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) |
2129 | { |
2126 | { |
2130 | u32 old_write_domain, old_read_domains; |
2127 | u32 old_write_domain, old_read_domains; |
2131 | 2128 | ||
2132 | /* Act a barrier for all accesses through the GTT */ |
2129 | /* Act a barrier for all accesses through the GTT */ |
2133 | mb(); |
2130 | mb(); |
2134 | 2131 | ||
2135 | /* Force a pagefault for domain tracking on next user access */ |
2132 | /* Force a pagefault for domain tracking on next user access */ |
2136 | // i915_gem_release_mmap(obj); |
2133 | // i915_gem_release_mmap(obj); |
2137 | 2134 | ||
2138 | if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) |
2135 | if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) |
2139 | return; |
2136 | return; |
2140 | 2137 | ||
2141 | old_read_domains = obj->base.read_domains; |
2138 | old_read_domains = obj->base.read_domains; |
2142 | old_write_domain = obj->base.write_domain; |
2139 | old_write_domain = obj->base.write_domain; |
2143 | 2140 | ||
2144 | obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; |
2141 | obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; |
2145 | obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; |
2142 | obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; |
2146 | 2143 | ||
2147 | trace_i915_gem_object_change_domain(obj, |
2144 | trace_i915_gem_object_change_domain(obj, |
2148 | old_read_domains, |
2145 | old_read_domains, |
2149 | old_write_domain); |
2146 | old_write_domain); |
2150 | } |
2147 | } |
2151 | 2148 | ||
2152 | /** |
2149 | /** |
2153 | * Unbinds an object from the GTT aperture. |
2150 | * Unbinds an object from the GTT aperture. |
2154 | */ |
2151 | */ |
2155 | int |
2152 | int |
2156 | i915_gem_object_unbind(struct drm_i915_gem_object *obj) |
2153 | i915_gem_object_unbind(struct drm_i915_gem_object *obj) |
2157 | { |
2154 | { |
2158 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
2155 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
2159 | int ret = 0; |
2156 | int ret = 0; |
2160 | 2157 | ||
2161 | if(obj == get_fb_obj()) |
2158 | if(obj == get_fb_obj()) |
2162 | return 0; |
2159 | return 0; |
2163 | 2160 | ||
2164 | if (obj->gtt_space == NULL) |
2161 | if (obj->gtt_space == NULL) |
2165 | return 0; |
2162 | return 0; |
2166 | 2163 | ||
2167 | if (obj->pin_count) |
2164 | if (obj->pin_count) |
2168 | return -EBUSY; |
2165 | return -EBUSY; |
2169 | 2166 | ||
2170 | BUG_ON(obj->pages == NULL); |
2167 | BUG_ON(obj->pages == NULL); |
2171 | 2168 | ||
2172 | ret = i915_gem_object_finish_gpu(obj); |
2169 | ret = i915_gem_object_finish_gpu(obj); |
2173 | if (ret) |
2170 | if (ret) |
2174 | return ret; |
2171 | return ret; |
2175 | /* Continue on if we fail due to EIO, the GPU is hung so we |
2172 | /* Continue on if we fail due to EIO, the GPU is hung so we |
2176 | * should be safe and we need to cleanup or else we might |
2173 | * should be safe and we need to cleanup or else we might |
2177 | * cause memory corruption through use-after-free. |
2174 | * cause memory corruption through use-after-free. |
2178 | */ |
2175 | */ |
2179 | 2176 | ||
2180 | i915_gem_object_finish_gtt(obj); |
2177 | i915_gem_object_finish_gtt(obj); |
2181 | 2178 | ||
2182 | /* release the fence reg _after_ flushing */ |
2179 | /* release the fence reg _after_ flushing */ |
2183 | ret = i915_gem_object_put_fence(obj); |
2180 | ret = i915_gem_object_put_fence(obj); |
2184 | if (ret) |
2181 | if (ret) |
2185 | return ret; |
2182 | return ret; |
2186 | 2183 | ||
2187 | trace_i915_gem_object_unbind(obj); |
2184 | trace_i915_gem_object_unbind(obj); |
2188 | 2185 | ||
2189 | if (obj->has_global_gtt_mapping) |
2186 | if (obj->has_global_gtt_mapping) |
2190 | i915_gem_gtt_unbind_object(obj); |
2187 | i915_gem_gtt_unbind_object(obj); |
2191 | if (obj->has_aliasing_ppgtt_mapping) { |
2188 | if (obj->has_aliasing_ppgtt_mapping) { |
2192 | i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); |
2189 | i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); |
2193 | obj->has_aliasing_ppgtt_mapping = 0; |
2190 | obj->has_aliasing_ppgtt_mapping = 0; |
2194 | } |
2191 | } |
2195 | i915_gem_gtt_finish_object(obj); |
2192 | i915_gem_gtt_finish_object(obj); |
2196 | 2193 | ||
2197 | list_del(&obj->mm_list); |
2194 | list_del(&obj->mm_list); |
2198 | list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
2195 | list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
2199 | /* Avoid an unnecessary call to unbind on rebind. */ |
2196 | /* Avoid an unnecessary call to unbind on rebind. */ |
2200 | obj->map_and_fenceable = true; |
2197 | obj->map_and_fenceable = true; |
2201 | 2198 | ||
2202 | drm_mm_put_block(obj->gtt_space); |
2199 | drm_mm_put_block(obj->gtt_space); |
2203 | obj->gtt_space = NULL; |
2200 | obj->gtt_space = NULL; |
2204 | obj->gtt_offset = 0; |
2201 | obj->gtt_offset = 0; |
2205 | 2202 | ||
2206 | return 0; |
2203 | return 0; |
2207 | } |
2204 | } |
2208 | 2205 | ||
2209 | int i915_gpu_idle(struct drm_device *dev) |
2206 | int i915_gpu_idle(struct drm_device *dev) |
2210 | { |
2207 | { |
2211 | drm_i915_private_t *dev_priv = dev->dev_private; |
2208 | drm_i915_private_t *dev_priv = dev->dev_private; |
2212 | struct intel_ring_buffer *ring; |
2209 | struct intel_ring_buffer *ring; |
2213 | int ret, i; |
2210 | int ret, i; |
2214 | 2211 | ||
2215 | /* Flush everything onto the inactive list. */ |
2212 | /* Flush everything onto the inactive list. */ |
2216 | for_each_ring(ring, dev_priv, i) { |
2213 | for_each_ring(ring, dev_priv, i) { |
2217 | ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); |
2214 | ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); |
2218 | if (ret) |
2215 | if (ret) |
2219 | return ret; |
2216 | return ret; |
2220 | 2217 | ||
2221 | ret = intel_ring_idle(ring); |
2218 | ret = intel_ring_idle(ring); |
2222 | if (ret) |
2219 | if (ret) |
2223 | return ret; |
2220 | return ret; |
2224 | } |
2221 | } |
2225 | 2222 | ||
2226 | return 0; |
2223 | return 0; |
2227 | } |
2224 | } |
2228 | 2225 | ||
2229 | static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, |
2226 | static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, |
2230 | struct drm_i915_gem_object *obj) |
2227 | struct drm_i915_gem_object *obj) |
2231 | { |
2228 | { |
2232 | drm_i915_private_t *dev_priv = dev->dev_private; |
2229 | drm_i915_private_t *dev_priv = dev->dev_private; |
2233 | uint64_t val; |
2230 | uint64_t val; |
2234 | 2231 | ||
2235 | if (obj) { |
2232 | if (obj) { |
2236 | u32 size = obj->gtt_space->size; |
2233 | u32 size = obj->gtt_space->size; |
2237 | 2234 | ||
2238 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
2235 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
2239 | 0xfffff000) << 32; |
2236 | 0xfffff000) << 32; |
2240 | val |= obj->gtt_offset & 0xfffff000; |
2237 | val |= obj->gtt_offset & 0xfffff000; |
2241 | val |= (uint64_t)((obj->stride / 128) - 1) << |
2238 | val |= (uint64_t)((obj->stride / 128) - 1) << |
2242 | SANDYBRIDGE_FENCE_PITCH_SHIFT; |
2239 | SANDYBRIDGE_FENCE_PITCH_SHIFT; |
2243 | 2240 | ||
2244 | if (obj->tiling_mode == I915_TILING_Y) |
2241 | if (obj->tiling_mode == I915_TILING_Y) |
2245 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
2242 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
2246 | val |= I965_FENCE_REG_VALID; |
2243 | val |= I965_FENCE_REG_VALID; |
2247 | } else |
2244 | } else |
2248 | val = 0; |
2245 | val = 0; |
2249 | 2246 | ||
2250 | I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); |
2247 | I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); |
2251 | POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); |
2248 | POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); |
2252 | } |
2249 | } |
2253 | 2250 | ||
2254 | static void i965_write_fence_reg(struct drm_device *dev, int reg, |
2251 | static void i965_write_fence_reg(struct drm_device *dev, int reg, |
2255 | struct drm_i915_gem_object *obj) |
2252 | struct drm_i915_gem_object *obj) |
2256 | { |
2253 | { |
2257 | drm_i915_private_t *dev_priv = dev->dev_private; |
2254 | drm_i915_private_t *dev_priv = dev->dev_private; |
2258 | uint64_t val; |
2255 | uint64_t val; |
2259 | 2256 | ||
2260 | if (obj) { |
2257 | if (obj) { |
2261 | u32 size = obj->gtt_space->size; |
2258 | u32 size = obj->gtt_space->size; |
2262 | 2259 | ||
2263 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
2260 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
2264 | 0xfffff000) << 32; |
2261 | 0xfffff000) << 32; |
2265 | val |= obj->gtt_offset & 0xfffff000; |
2262 | val |= obj->gtt_offset & 0xfffff000; |
2266 | val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; |
2263 | val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; |
2267 | if (obj->tiling_mode == I915_TILING_Y) |
2264 | if (obj->tiling_mode == I915_TILING_Y) |
2268 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
2265 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
2269 | val |= I965_FENCE_REG_VALID; |
2266 | val |= I965_FENCE_REG_VALID; |
2270 | } else |
2267 | } else |
2271 | val = 0; |
2268 | val = 0; |
2272 | 2269 | ||
2273 | I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); |
2270 | I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); |
2274 | POSTING_READ(FENCE_REG_965_0 + reg * 8); |
2271 | POSTING_READ(FENCE_REG_965_0 + reg * 8); |
2275 | } |
2272 | } |
2276 | 2273 | ||
2277 | static void i915_write_fence_reg(struct drm_device *dev, int reg, |
2274 | static void i915_write_fence_reg(struct drm_device *dev, int reg, |
2278 | struct drm_i915_gem_object *obj) |
2275 | struct drm_i915_gem_object *obj) |
2279 | { |
2276 | { |
2280 | drm_i915_private_t *dev_priv = dev->dev_private; |
2277 | drm_i915_private_t *dev_priv = dev->dev_private; |
2281 | u32 val; |
2278 | u32 val; |
2282 | 2279 | ||
2283 | if (obj) { |
2280 | if (obj) { |
2284 | u32 size = obj->gtt_space->size; |
2281 | u32 size = obj->gtt_space->size; |
2285 | int pitch_val; |
2282 | int pitch_val; |
2286 | int tile_width; |
2283 | int tile_width; |
2287 | 2284 | ||
2288 | WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || |
2285 | WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || |
2289 | (size & -size) != size || |
2286 | (size & -size) != size || |
2290 | (obj->gtt_offset & (size - 1)), |
2287 | (obj->gtt_offset & (size - 1)), |
2291 | "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", |
2288 | "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", |
2292 | obj->gtt_offset, obj->map_and_fenceable, size); |
2289 | obj->gtt_offset, obj->map_and_fenceable, size); |
2293 | 2290 | ||
2294 | if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) |
2291 | if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) |
2295 | tile_width = 128; |
2292 | tile_width = 128; |
2296 | else |
2293 | else |
2297 | tile_width = 512; |
2294 | tile_width = 512; |
2298 | 2295 | ||
2299 | /* Note: pitch better be a power of two tile widths */ |
2296 | /* Note: pitch better be a power of two tile widths */ |
2300 | pitch_val = obj->stride / tile_width; |
2297 | pitch_val = obj->stride / tile_width; |
2301 | pitch_val = ffs(pitch_val) - 1; |
2298 | pitch_val = ffs(pitch_val) - 1; |
2302 | 2299 | ||
2303 | val = obj->gtt_offset; |
2300 | val = obj->gtt_offset; |
2304 | if (obj->tiling_mode == I915_TILING_Y) |
2301 | if (obj->tiling_mode == I915_TILING_Y) |
2305 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
2302 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
2306 | val |= I915_FENCE_SIZE_BITS(size); |
2303 | val |= I915_FENCE_SIZE_BITS(size); |
2307 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
2304 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
2308 | val |= I830_FENCE_REG_VALID; |
2305 | val |= I830_FENCE_REG_VALID; |
2309 | } else |
2306 | } else |
2310 | val = 0; |
2307 | val = 0; |
2311 | 2308 | ||
2312 | if (reg < 8) |
2309 | if (reg < 8) |
2313 | reg = FENCE_REG_830_0 + reg * 4; |
2310 | reg = FENCE_REG_830_0 + reg * 4; |
2314 | else |
2311 | else |
2315 | reg = FENCE_REG_945_8 + (reg - 8) * 4; |
2312 | reg = FENCE_REG_945_8 + (reg - 8) * 4; |
2316 | 2313 | ||
2317 | I915_WRITE(reg, val); |
2314 | I915_WRITE(reg, val); |
2318 | POSTING_READ(reg); |
2315 | POSTING_READ(reg); |
2319 | } |
2316 | } |
2320 | 2317 | ||
2321 | static void i830_write_fence_reg(struct drm_device *dev, int reg, |
2318 | static void i830_write_fence_reg(struct drm_device *dev, int reg, |
2322 | struct drm_i915_gem_object *obj) |
2319 | struct drm_i915_gem_object *obj) |
2323 | { |
2320 | { |
2324 | drm_i915_private_t *dev_priv = dev->dev_private; |
2321 | drm_i915_private_t *dev_priv = dev->dev_private; |
2325 | uint32_t val; |
2322 | uint32_t val; |
2326 | 2323 | ||
2327 | if (obj) { |
2324 | if (obj) { |
2328 | u32 size = obj->gtt_space->size; |
2325 | u32 size = obj->gtt_space->size; |
2329 | uint32_t pitch_val; |
2326 | uint32_t pitch_val; |
2330 | 2327 | ||
2331 | WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || |
2328 | WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || |
2332 | (size & -size) != size || |
2329 | (size & -size) != size || |
2333 | (obj->gtt_offset & (size - 1)), |
2330 | (obj->gtt_offset & (size - 1)), |
2334 | "object 0x%08x not 512K or pot-size 0x%08x aligned\n", |
2331 | "object 0x%08x not 512K or pot-size 0x%08x aligned\n", |
2335 | obj->gtt_offset, size); |
2332 | obj->gtt_offset, size); |
2336 | 2333 | ||
2337 | pitch_val = obj->stride / 128; |
2334 | pitch_val = obj->stride / 128; |
2338 | pitch_val = ffs(pitch_val) - 1; |
2335 | pitch_val = ffs(pitch_val) - 1; |
2339 | 2336 | ||
2340 | val = obj->gtt_offset; |
2337 | val = obj->gtt_offset; |
2341 | if (obj->tiling_mode == I915_TILING_Y) |
2338 | if (obj->tiling_mode == I915_TILING_Y) |
2342 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
2339 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
2343 | val |= I830_FENCE_SIZE_BITS(size); |
2340 | val |= I830_FENCE_SIZE_BITS(size); |
2344 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
2341 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
2345 | val |= I830_FENCE_REG_VALID; |
2342 | val |= I830_FENCE_REG_VALID; |
2346 | } else |
2343 | } else |
2347 | val = 0; |
2344 | val = 0; |
2348 | 2345 | ||
2349 | I915_WRITE(FENCE_REG_830_0 + reg * 4, val); |
2346 | I915_WRITE(FENCE_REG_830_0 + reg * 4, val); |
2350 | POSTING_READ(FENCE_REG_830_0 + reg * 4); |
2347 | POSTING_READ(FENCE_REG_830_0 + reg * 4); |
2351 | } |
2348 | } |
2352 | 2349 | ||
2353 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
2350 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
2354 | struct drm_i915_gem_object *obj) |
2351 | struct drm_i915_gem_object *obj) |
2355 | { |
2352 | { |
2356 | switch (INTEL_INFO(dev)->gen) { |
2353 | switch (INTEL_INFO(dev)->gen) { |
2357 | case 7: |
2354 | case 7: |
2358 | case 6: sandybridge_write_fence_reg(dev, reg, obj); break; |
2355 | case 6: sandybridge_write_fence_reg(dev, reg, obj); break; |
2359 | case 5: |
2356 | case 5: |
2360 | case 4: i965_write_fence_reg(dev, reg, obj); break; |
2357 | case 4: i965_write_fence_reg(dev, reg, obj); break; |
2361 | case 3: i915_write_fence_reg(dev, reg, obj); break; |
2358 | case 3: i915_write_fence_reg(dev, reg, obj); break; |
2362 | case 2: i830_write_fence_reg(dev, reg, obj); break; |
2359 | case 2: i830_write_fence_reg(dev, reg, obj); break; |
2363 | default: break; |
2360 | default: break; |
2364 | } |
2361 | } |
2365 | } |
2362 | } |
2366 | 2363 | ||
2367 | static inline int fence_number(struct drm_i915_private *dev_priv, |
2364 | static inline int fence_number(struct drm_i915_private *dev_priv, |
2368 | struct drm_i915_fence_reg *fence) |
2365 | struct drm_i915_fence_reg *fence) |
2369 | { |
2366 | { |
2370 | return fence - dev_priv->fence_regs; |
2367 | return fence - dev_priv->fence_regs; |
2371 | } |
2368 | } |
2372 | 2369 | ||
2373 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
2370 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
2374 | struct drm_i915_fence_reg *fence, |
2371 | struct drm_i915_fence_reg *fence, |
2375 | bool enable) |
2372 | bool enable) |
2376 | { |
2373 | { |
2377 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2374 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2378 | int reg = fence_number(dev_priv, fence); |
2375 | int reg = fence_number(dev_priv, fence); |
2379 | 2376 | ||
2380 | i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); |
2377 | i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); |
2381 | 2378 | ||
2382 | if (enable) { |
2379 | if (enable) { |
2383 | obj->fence_reg = reg; |
2380 | obj->fence_reg = reg; |
2384 | fence->obj = obj; |
2381 | fence->obj = obj; |
2385 | list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
2382 | list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
2386 | } else { |
2383 | } else { |
2387 | obj->fence_reg = I915_FENCE_REG_NONE; |
2384 | obj->fence_reg = I915_FENCE_REG_NONE; |
2388 | fence->obj = NULL; |
2385 | fence->obj = NULL; |
2389 | list_del_init(&fence->lru_list); |
2386 | list_del_init(&fence->lru_list); |
2390 | } |
2387 | } |
2391 | } |
2388 | } |
2392 | 2389 | ||
2393 | static int |
2390 | static int |
2394 | i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) |
2391 | i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) |
2395 | { |
2392 | { |
2396 | if (obj->last_fenced_seqno) { |
2393 | if (obj->last_fenced_seqno) { |
2397 | int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); |
2394 | int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); |
2398 | if (ret) |
2395 | if (ret) |
2399 | return ret; |
2396 | return ret; |
2400 | 2397 | ||
2401 | obj->last_fenced_seqno = 0; |
2398 | obj->last_fenced_seqno = 0; |
2402 | } |
2399 | } |
2403 | 2400 | ||
2404 | /* Ensure that all CPU reads are completed before installing a fence |
2401 | /* Ensure that all CPU reads are completed before installing a fence |
2405 | * and all writes before removing the fence. |
2402 | * and all writes before removing the fence. |
2406 | */ |
2403 | */ |
2407 | if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) |
2404 | if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) |
2408 | mb(); |
2405 | mb(); |
2409 | 2406 | ||
2410 | obj->fenced_gpu_access = false; |
2407 | obj->fenced_gpu_access = false; |
2411 | return 0; |
2408 | return 0; |
2412 | } |
2409 | } |
2413 | 2410 | ||
2414 | int |
2411 | int |
2415 | i915_gem_object_put_fence(struct drm_i915_gem_object *obj) |
2412 | i915_gem_object_put_fence(struct drm_i915_gem_object *obj) |
2416 | { |
2413 | { |
2417 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2414 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2418 | int ret; |
2415 | int ret; |
2419 | 2416 | ||
2420 | ret = i915_gem_object_flush_fence(obj); |
2417 | ret = i915_gem_object_flush_fence(obj); |
2421 | if (ret) |
2418 | if (ret) |
2422 | return ret; |
2419 | return ret; |
2423 | 2420 | ||
2424 | if (obj->fence_reg == I915_FENCE_REG_NONE) |
2421 | if (obj->fence_reg == I915_FENCE_REG_NONE) |
2425 | return 0; |
2422 | return 0; |
2426 | 2423 | ||
2427 | i915_gem_object_update_fence(obj, |
2424 | i915_gem_object_update_fence(obj, |
2428 | &dev_priv->fence_regs[obj->fence_reg], |
2425 | &dev_priv->fence_regs[obj->fence_reg], |
2429 | false); |
2426 | false); |
2430 | i915_gem_object_fence_lost(obj); |
2427 | i915_gem_object_fence_lost(obj); |
2431 | 2428 | ||
2432 | return 0; |
2429 | return 0; |
2433 | } |
2430 | } |
2434 | 2431 | ||
2435 | static struct drm_i915_fence_reg * |
2432 | static struct drm_i915_fence_reg * |
2436 | i915_find_fence_reg(struct drm_device *dev) |
2433 | i915_find_fence_reg(struct drm_device *dev) |
2437 | { |
2434 | { |
2438 | struct drm_i915_private *dev_priv = dev->dev_private; |
2435 | struct drm_i915_private *dev_priv = dev->dev_private; |
2439 | struct drm_i915_fence_reg *reg, *avail; |
2436 | struct drm_i915_fence_reg *reg, *avail; |
2440 | int i; |
2437 | int i; |
2441 | 2438 | ||
2442 | /* First try to find a free reg */ |
2439 | /* First try to find a free reg */ |
2443 | avail = NULL; |
2440 | avail = NULL; |
2444 | for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { |
2441 | for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { |
2445 | reg = &dev_priv->fence_regs[i]; |
2442 | reg = &dev_priv->fence_regs[i]; |
2446 | if (!reg->obj) |
2443 | if (!reg->obj) |
2447 | return reg; |
2444 | return reg; |
2448 | 2445 | ||
2449 | if (!reg->pin_count) |
2446 | if (!reg->pin_count) |
2450 | avail = reg; |
2447 | avail = reg; |
2451 | } |
2448 | } |
2452 | 2449 | ||
2453 | if (avail == NULL) |
2450 | if (avail == NULL) |
2454 | return NULL; |
2451 | return NULL; |
2455 | 2452 | ||
2456 | /* None available, try to steal one or wait for a user to finish */ |
2453 | /* None available, try to steal one or wait for a user to finish */ |
2457 | list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { |
2454 | list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { |
2458 | if (reg->pin_count) |
2455 | if (reg->pin_count) |
2459 | continue; |
2456 | continue; |
2460 | 2457 | ||
2461 | return reg; |
2458 | return reg; |
2462 | } |
2459 | } |
2463 | 2460 | ||
2464 | return NULL; |
2461 | return NULL; |
2465 | } |
2462 | } |
2466 | 2463 | ||
2467 | /** |
2464 | /** |
2468 | * i915_gem_object_get_fence - set up fencing for an object |
2465 | * i915_gem_object_get_fence - set up fencing for an object |
2469 | * @obj: object to map through a fence reg |
2466 | * @obj: object to map through a fence reg |
2470 | * |
2467 | * |
2471 | * When mapping objects through the GTT, userspace wants to be able to write |
2468 | * When mapping objects through the GTT, userspace wants to be able to write |
2472 | * to them without having to worry about swizzling if the object is tiled. |
2469 | * to them without having to worry about swizzling if the object is tiled. |
2473 | * This function walks the fence regs looking for a free one for @obj, |
2470 | * This function walks the fence regs looking for a free one for @obj, |
2474 | * stealing one if it can't find any. |
2471 | * stealing one if it can't find any. |
2475 | * |
2472 | * |
2476 | * It then sets up the reg based on the object's properties: address, pitch |
2473 | * It then sets up the reg based on the object's properties: address, pitch |
2477 | * and tiling format. |
2474 | * and tiling format. |
2478 | * |
2475 | * |
2479 | * For an untiled surface, this removes any existing fence. |
2476 | * For an untiled surface, this removes any existing fence. |
2480 | */ |
2477 | */ |
2481 | int |
2478 | int |
2482 | i915_gem_object_get_fence(struct drm_i915_gem_object *obj) |
2479 | i915_gem_object_get_fence(struct drm_i915_gem_object *obj) |
2483 | { |
2480 | { |
2484 | struct drm_device *dev = obj->base.dev; |
2481 | struct drm_device *dev = obj->base.dev; |
2485 | struct drm_i915_private *dev_priv = dev->dev_private; |
2482 | struct drm_i915_private *dev_priv = dev->dev_private; |
2486 | bool enable = obj->tiling_mode != I915_TILING_NONE; |
2483 | bool enable = obj->tiling_mode != I915_TILING_NONE; |
2487 | struct drm_i915_fence_reg *reg; |
2484 | struct drm_i915_fence_reg *reg; |
2488 | int ret; |
2485 | int ret; |
2489 | 2486 | ||
2490 | /* Have we updated the tiling parameters upon the object and so |
2487 | /* Have we updated the tiling parameters upon the object and so |
2491 | * will need to serialise the write to the associated fence register? |
2488 | * will need to serialise the write to the associated fence register? |
2492 | */ |
2489 | */ |
2493 | if (obj->fence_dirty) { |
2490 | if (obj->fence_dirty) { |
2494 | ret = i915_gem_object_flush_fence(obj); |
2491 | ret = i915_gem_object_flush_fence(obj); |
2495 | if (ret) |
2492 | if (ret) |
2496 | return ret; |
2493 | return ret; |
2497 | } |
2494 | } |
2498 | 2495 | ||
2499 | /* Just update our place in the LRU if our fence is getting reused. */ |
2496 | /* Just update our place in the LRU if our fence is getting reused. */ |
2500 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
2497 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
2501 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
2498 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
2502 | if (!obj->fence_dirty) { |
2499 | if (!obj->fence_dirty) { |
2503 | list_move_tail(®->lru_list, |
2500 | list_move_tail(®->lru_list, |
2504 | &dev_priv->mm.fence_list); |
2501 | &dev_priv->mm.fence_list); |
2505 | return 0; |
2502 | return 0; |
2506 | } |
2503 | } |
2507 | } else if (enable) { |
2504 | } else if (enable) { |
2508 | reg = i915_find_fence_reg(dev); |
2505 | reg = i915_find_fence_reg(dev); |
2509 | if (reg == NULL) |
2506 | if (reg == NULL) |
2510 | return -EDEADLK; |
2507 | return -EDEADLK; |
2511 | 2508 | ||
2512 | if (reg->obj) { |
2509 | if (reg->obj) { |
2513 | struct drm_i915_gem_object *old = reg->obj; |
2510 | struct drm_i915_gem_object *old = reg->obj; |
2514 | 2511 | ||
2515 | ret = i915_gem_object_flush_fence(old); |
2512 | ret = i915_gem_object_flush_fence(old); |
2516 | if (ret) |
2513 | if (ret) |
2517 | return ret; |
2514 | return ret; |
2518 | 2515 | ||
2519 | i915_gem_object_fence_lost(old); |
2516 | i915_gem_object_fence_lost(old); |
2520 | } |
2517 | } |
2521 | } else |
2518 | } else |
2522 | return 0; |
2519 | return 0; |
2523 | 2520 | ||
2524 | i915_gem_object_update_fence(obj, reg, enable); |
2521 | i915_gem_object_update_fence(obj, reg, enable); |
2525 | obj->fence_dirty = false; |
2522 | obj->fence_dirty = false; |
2526 | 2523 | ||
2527 | return 0; |
2524 | return 0; |
2528 | } |
2525 | } |
2529 | 2526 | ||
2530 | static bool i915_gem_valid_gtt_space(struct drm_device *dev, |
2527 | static bool i915_gem_valid_gtt_space(struct drm_device *dev, |
2531 | struct drm_mm_node *gtt_space, |
2528 | struct drm_mm_node *gtt_space, |
2532 | unsigned long cache_level) |
2529 | unsigned long cache_level) |
2533 | { |
2530 | { |
2534 | struct drm_mm_node *other; |
2531 | struct drm_mm_node *other; |
2535 | 2532 | ||
2536 | /* On non-LLC machines we have to be careful when putting differing |
2533 | /* On non-LLC machines we have to be careful when putting differing |
2537 | * types of snoopable memory together to avoid the prefetcher |
2534 | * types of snoopable memory together to avoid the prefetcher |
2538 | * crossing memory domains and dieing. |
2535 | * crossing memory domains and dieing. |
2539 | */ |
2536 | */ |
2540 | if (HAS_LLC(dev)) |
2537 | if (HAS_LLC(dev)) |
2541 | return true; |
2538 | return true; |
2542 | 2539 | ||
2543 | if (gtt_space == NULL) |
2540 | if (gtt_space == NULL) |
2544 | return true; |
2541 | return true; |
2545 | 2542 | ||
2546 | if (list_empty(>t_space->node_list)) |
2543 | if (list_empty(>t_space->node_list)) |
2547 | return true; |
2544 | return true; |
2548 | 2545 | ||
2549 | other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); |
2546 | other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); |
2550 | if (other->allocated && !other->hole_follows && other->color != cache_level) |
2547 | if (other->allocated && !other->hole_follows && other->color != cache_level) |
2551 | return false; |
2548 | return false; |
2552 | 2549 | ||
2553 | other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); |
2550 | other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); |
2554 | if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) |
2551 | if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) |
2555 | return false; |
2552 | return false; |
2556 | 2553 | ||
2557 | return true; |
2554 | return true; |
2558 | } |
2555 | } |
2559 | 2556 | ||
2560 | static void i915_gem_verify_gtt(struct drm_device *dev) |
2557 | static void i915_gem_verify_gtt(struct drm_device *dev) |
2561 | { |
2558 | { |
2562 | #if WATCH_GTT |
2559 | #if WATCH_GTT |
2563 | struct drm_i915_private *dev_priv = dev->dev_private; |
2560 | struct drm_i915_private *dev_priv = dev->dev_private; |
2564 | struct drm_i915_gem_object *obj; |
2561 | struct drm_i915_gem_object *obj; |
2565 | int err = 0; |
2562 | int err = 0; |
2566 | 2563 | ||
2567 | list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { |
2564 | list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { |
2568 | if (obj->gtt_space == NULL) { |
2565 | if (obj->gtt_space == NULL) { |
2569 | printk(KERN_ERR "object found on GTT list with no space reserved\n"); |
2566 | printk(KERN_ERR "object found on GTT list with no space reserved\n"); |
2570 | err++; |
2567 | err++; |
2571 | continue; |
2568 | continue; |
2572 | } |
2569 | } |
2573 | 2570 | ||
2574 | if (obj->cache_level != obj->gtt_space->color) { |
2571 | if (obj->cache_level != obj->gtt_space->color) { |
2575 | printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", |
2572 | printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", |
2576 | obj->gtt_space->start, |
2573 | obj->gtt_space->start, |
2577 | obj->gtt_space->start + obj->gtt_space->size, |
2574 | obj->gtt_space->start + obj->gtt_space->size, |
2578 | obj->cache_level, |
2575 | obj->cache_level, |
2579 | obj->gtt_space->color); |
2576 | obj->gtt_space->color); |
2580 | err++; |
2577 | err++; |
2581 | continue; |
2578 | continue; |
2582 | } |
2579 | } |
2583 | 2580 | ||
2584 | if (!i915_gem_valid_gtt_space(dev, |
2581 | if (!i915_gem_valid_gtt_space(dev, |
2585 | obj->gtt_space, |
2582 | obj->gtt_space, |
2586 | obj->cache_level)) { |
2583 | obj->cache_level)) { |
2587 | printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n", |
2584 | printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n", |
2588 | obj->gtt_space->start, |
2585 | obj->gtt_space->start, |
2589 | obj->gtt_space->start + obj->gtt_space->size, |
2586 | obj->gtt_space->start + obj->gtt_space->size, |
2590 | obj->cache_level); |
2587 | obj->cache_level); |
2591 | err++; |
2588 | err++; |
2592 | continue; |
2589 | continue; |
2593 | } |
2590 | } |
2594 | } |
2591 | } |
2595 | 2592 | ||
2596 | WARN_ON(err); |
2593 | WARN_ON(err); |
2597 | #endif |
2594 | #endif |
2598 | } |
2595 | } |
2599 | 2596 | ||
2600 | /** |
2597 | /** |
2601 | * Finds free space in the GTT aperture and binds the object there. |
2598 | * Finds free space in the GTT aperture and binds the object there. |
2602 | */ |
2599 | */ |
2603 | static int |
2600 | static int |
2604 | i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
2601 | i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
2605 | unsigned alignment, |
2602 | unsigned alignment, |
2606 | bool map_and_fenceable, |
2603 | bool map_and_fenceable, |
2607 | bool nonblocking) |
2604 | bool nonblocking) |
2608 | { |
2605 | { |
2609 | struct drm_device *dev = obj->base.dev; |
2606 | struct drm_device *dev = obj->base.dev; |
2610 | drm_i915_private_t *dev_priv = dev->dev_private; |
2607 | drm_i915_private_t *dev_priv = dev->dev_private; |
2611 | struct drm_mm_node *node; |
2608 | struct drm_mm_node *node; |
2612 | u32 size, fence_size, fence_alignment, unfenced_alignment; |
2609 | u32 size, fence_size, fence_alignment, unfenced_alignment; |
2613 | bool mappable, fenceable; |
2610 | bool mappable, fenceable; |
2614 | int ret; |
2611 | int ret; |
2615 | 2612 | ||
2616 | if (obj->madv != I915_MADV_WILLNEED) { |
2613 | if (obj->madv != I915_MADV_WILLNEED) { |
2617 | DRM_ERROR("Attempting to bind a purgeable object\n"); |
2614 | DRM_ERROR("Attempting to bind a purgeable object\n"); |
2618 | return -EINVAL; |
2615 | return -EINVAL; |
2619 | } |
2616 | } |
2620 | 2617 | ||
2621 | fence_size = i915_gem_get_gtt_size(dev, |
2618 | fence_size = i915_gem_get_gtt_size(dev, |
2622 | obj->base.size, |
2619 | obj->base.size, |
2623 | obj->tiling_mode); |
2620 | obj->tiling_mode); |
2624 | fence_alignment = i915_gem_get_gtt_alignment(dev, |
2621 | fence_alignment = i915_gem_get_gtt_alignment(dev, |
2625 | obj->base.size, |
2622 | obj->base.size, |
2626 | obj->tiling_mode); |
2623 | obj->tiling_mode); |
2627 | unfenced_alignment = |
2624 | unfenced_alignment = |
2628 | i915_gem_get_unfenced_gtt_alignment(dev, |
2625 | i915_gem_get_unfenced_gtt_alignment(dev, |
2629 | obj->base.size, |
2626 | obj->base.size, |
2630 | obj->tiling_mode); |
2627 | obj->tiling_mode); |
2631 | 2628 | ||
2632 | if (alignment == 0) |
2629 | if (alignment == 0) |
2633 | alignment = map_and_fenceable ? fence_alignment : |
2630 | alignment = map_and_fenceable ? fence_alignment : |
2634 | unfenced_alignment; |
2631 | unfenced_alignment; |
2635 | if (map_and_fenceable && alignment & (fence_alignment - 1)) { |
2632 | if (map_and_fenceable && alignment & (fence_alignment - 1)) { |
2636 | DRM_ERROR("Invalid object alignment requested %u\n", alignment); |
2633 | DRM_ERROR("Invalid object alignment requested %u\n", alignment); |
2637 | return -EINVAL; |
2634 | return -EINVAL; |
2638 | } |
2635 | } |
2639 | 2636 | ||
2640 | size = map_and_fenceable ? fence_size : obj->base.size; |
2637 | size = map_and_fenceable ? fence_size : obj->base.size; |
2641 | 2638 | ||
2642 | /* If the object is bigger than the entire aperture, reject it early |
2639 | /* If the object is bigger than the entire aperture, reject it early |
2643 | * before evicting everything in a vain attempt to find space. |
2640 | * before evicting everything in a vain attempt to find space. |
2644 | */ |
2641 | */ |
2645 | if (obj->base.size > |
2642 | if (obj->base.size > |
2646 | (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { |
2643 | (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { |
2647 | DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
2644 | DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
2648 | return -E2BIG; |
2645 | return -E2BIG; |
2649 | } |
2646 | } |
2650 | 2647 | ||
2651 | ret = i915_gem_object_get_pages(obj); |
2648 | ret = i915_gem_object_get_pages(obj); |
2652 | if (ret) |
2649 | if (ret) |
2653 | return ret; |
2650 | return ret; |
2654 | 2651 | ||
2655 | i915_gem_object_pin_pages(obj); |
2652 | i915_gem_object_pin_pages(obj); |
2656 | 2653 | ||
2657 | node = kzalloc(sizeof(*node), GFP_KERNEL); |
2654 | node = kzalloc(sizeof(*node), GFP_KERNEL); |
2658 | if (node == NULL) { |
2655 | if (node == NULL) { |
2659 | i915_gem_object_unpin_pages(obj); |
2656 | i915_gem_object_unpin_pages(obj); |
2660 | return -ENOMEM; |
2657 | return -ENOMEM; |
2661 | } |
2658 | } |
2662 | 2659 | ||
2663 | search_free: |
2660 | search_free: |
2664 | if (map_and_fenceable) |
2661 | if (map_and_fenceable) |
2665 | ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node, |
2662 | ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node, |
2666 | size, alignment, obj->cache_level, |
2663 | size, alignment, obj->cache_level, |
2667 | 0, dev_priv->mm.gtt_mappable_end); |
2664 | 0, dev_priv->mm.gtt_mappable_end); |
2668 | else |
2665 | else |
2669 | ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node, |
2666 | ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node, |
2670 | size, alignment, obj->cache_level); |
2667 | size, alignment, obj->cache_level); |
2671 | if (ret) { |
2668 | if (ret) { |
2672 | 2669 | ||
2673 | i915_gem_object_unpin_pages(obj); |
2670 | i915_gem_object_unpin_pages(obj); |
2674 | kfree(node); |
2671 | kfree(node); |
2675 | return ret; |
2672 | return ret; |
2676 | } |
2673 | } |
2677 | if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) { |
2674 | if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) { |
2678 | i915_gem_object_unpin_pages(obj); |
2675 | i915_gem_object_unpin_pages(obj); |
2679 | drm_mm_put_block(node); |
2676 | drm_mm_put_block(node); |
2680 | return -EINVAL; |
2677 | return -EINVAL; |
2681 | } |
2678 | } |
2682 | 2679 | ||
2683 | ret = i915_gem_gtt_prepare_object(obj); |
2680 | ret = i915_gem_gtt_prepare_object(obj); |
2684 | if (ret) { |
2681 | if (ret) { |
2685 | i915_gem_object_unpin_pages(obj); |
2682 | i915_gem_object_unpin_pages(obj); |
2686 | drm_mm_put_block(node); |
2683 | drm_mm_put_block(node); |
2687 | return ret; |
2684 | return ret; |
2688 | } |
2685 | } |
2689 | 2686 | ||
2690 | list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list); |
2687 | list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list); |
2691 | list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
2688 | list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
2692 | 2689 | ||
2693 | obj->gtt_space = node; |
2690 | obj->gtt_space = node; |
2694 | obj->gtt_offset = node->start; |
2691 | obj->gtt_offset = node->start; |
2695 | 2692 | ||
2696 | fenceable = |
2693 | fenceable = |
2697 | node->size == fence_size && |
2694 | node->size == fence_size && |
2698 | (node->start & (fence_alignment - 1)) == 0; |
2695 | (node->start & (fence_alignment - 1)) == 0; |
2699 | 2696 | ||
2700 | mappable = |
2697 | mappable = |
2701 | obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; |
2698 | obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; |
2702 | 2699 | ||
2703 | obj->map_and_fenceable = mappable && fenceable; |
2700 | obj->map_and_fenceable = mappable && fenceable; |
2704 | 2701 | ||
2705 | i915_gem_object_unpin_pages(obj); |
2702 | i915_gem_object_unpin_pages(obj); |
2706 | trace_i915_gem_object_bind(obj, map_and_fenceable); |
2703 | trace_i915_gem_object_bind(obj, map_and_fenceable); |
2707 | i915_gem_verify_gtt(dev); |
2704 | i915_gem_verify_gtt(dev); |
2708 | return 0; |
2705 | return 0; |
2709 | } |
2706 | } |
2710 | 2707 | ||
2711 | void |
2708 | void |
2712 | i915_gem_clflush_object(struct drm_i915_gem_object *obj) |
2709 | i915_gem_clflush_object(struct drm_i915_gem_object *obj) |
2713 | { |
2710 | { |
2714 | /* If we don't have a page list set up, then we're not pinned |
2711 | /* If we don't have a page list set up, then we're not pinned |
2715 | * to GPU, and we can ignore the cache flush because it'll happen |
2712 | * to GPU, and we can ignore the cache flush because it'll happen |
2716 | * again at bind time. |
2713 | * again at bind time. |
2717 | */ |
2714 | */ |
2718 | if (obj->pages == NULL) |
2715 | if (obj->pages == NULL) |
2719 | return; |
2716 | return; |
2720 | 2717 | ||
2721 | /* If the GPU is snooping the contents of the CPU cache, |
2718 | /* If the GPU is snooping the contents of the CPU cache, |
2722 | * we do not need to manually clear the CPU cache lines. However, |
2719 | * we do not need to manually clear the CPU cache lines. However, |
2723 | * the caches are only snooped when the render cache is |
2720 | * the caches are only snooped when the render cache is |
2724 | * flushed/invalidated. As we always have to emit invalidations |
2721 | * flushed/invalidated. As we always have to emit invalidations |
2725 | * and flushes when moving into and out of the RENDER domain, correct |
2722 | * and flushes when moving into and out of the RENDER domain, correct |
2726 | * snooping behaviour occurs naturally as the result of our domain |
2723 | * snooping behaviour occurs naturally as the result of our domain |
2727 | * tracking. |
2724 | * tracking. |
2728 | */ |
2725 | */ |
2729 | if (obj->cache_level != I915_CACHE_NONE) |
2726 | if (obj->cache_level != I915_CACHE_NONE) |
2730 | return; |
2727 | return; |
2731 | #if 0 |
2728 | #if 0 |
2732 | if(obj->mapped != NULL) |
2729 | if(obj->mapped != NULL) |
2733 | { |
2730 | { |
2734 | uint8_t *page_virtual; |
2731 | uint8_t *page_virtual; |
2735 | unsigned int i; |
2732 | unsigned int i; |
2736 | 2733 | ||
2737 | page_virtual = obj->mapped; |
2734 | page_virtual = obj->mapped; |
2738 | asm volatile("mfence"); |
2735 | asm volatile("mfence"); |
2739 | for (i = 0; i < obj->base.size; i += x86_clflush_size) |
2736 | for (i = 0; i < obj->base.size; i += x86_clflush_size) |
2740 | clflush(page_virtual + i); |
2737 | clflush(page_virtual + i); |
2741 | asm volatile("mfence"); |
2738 | asm volatile("mfence"); |
2742 | } |
2739 | } |
2743 | else |
2740 | else |
2744 | { |
2741 | { |
2745 | uint8_t *page_virtual; |
2742 | uint8_t *page_virtual; |
2746 | unsigned int i; |
2743 | unsigned int i; |
2747 | page_virtual = AllocKernelSpace(obj->base.size); |
2744 | page_virtual = AllocKernelSpace(obj->base.size); |
2748 | if(page_virtual != NULL) |
2745 | if(page_virtual != NULL) |
2749 | { |
2746 | { |
2750 | dma_addr_t *src, *dst; |
2747 | dma_addr_t *src, *dst; |
2751 | u32 count; |
2748 | u32 count; |
2752 | 2749 | ||
2753 | #define page_tabs 0xFDC00000 /* really dirty hack */ |
2750 | #define page_tabs 0xFDC00000 /* really dirty hack */ |
2754 | 2751 | ||
2755 | src = obj->pages.page; |
2752 | src = obj->pages.page; |
2756 | dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12]; |
2753 | dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12]; |
2757 | count = obj->base.size/4096; |
2754 | count = obj->base.size/4096; |
2758 | 2755 | ||
2759 | while(count--) |
2756 | while(count--) |
2760 | { |
2757 | { |
2761 | *dst++ = (0xFFFFF000 & *src++) | 0x001 ; |
2758 | *dst++ = (0xFFFFF000 & *src++) | 0x001 ; |
2762 | }; |
2759 | }; |
2763 | 2760 | ||
2764 | asm volatile("mfence"); |
2761 | asm volatile("mfence"); |
2765 | for (i = 0; i < obj->base.size; i += x86_clflush_size) |
2762 | for (i = 0; i < obj->base.size; i += x86_clflush_size) |
2766 | clflush(page_virtual + i); |
2763 | clflush(page_virtual + i); |
2767 | asm volatile("mfence"); |
2764 | asm volatile("mfence"); |
2768 | FreeKernelSpace(page_virtual); |
2765 | FreeKernelSpace(page_virtual); |
2769 | } |
2766 | } |
2770 | else |
2767 | else |
2771 | { |
2768 | { |
2772 | asm volatile ( |
2769 | asm volatile ( |
2773 | "mfence \n" |
2770 | "mfence \n" |
2774 | "wbinvd \n" /* this is really ugly */ |
2771 | "wbinvd \n" /* this is really ugly */ |
2775 | "mfence"); |
2772 | "mfence"); |
2776 | } |
2773 | } |
2777 | } |
2774 | } |
2778 | #endif |
2775 | #endif |
2779 | 2776 | ||
2780 | } |
2777 | } |
2781 | 2778 | ||
2782 | /** Flushes the GTT write domain for the object if it's dirty. */ |
2779 | /** Flushes the GTT write domain for the object if it's dirty. */ |
2783 | static void |
2780 | static void |
2784 | i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) |
2781 | i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) |
2785 | { |
2782 | { |
2786 | uint32_t old_write_domain; |
2783 | uint32_t old_write_domain; |
2787 | 2784 | ||
2788 | if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) |
2785 | if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) |
2789 | return; |
2786 | return; |
2790 | 2787 | ||
2791 | /* No actual flushing is required for the GTT write domain. Writes |
2788 | /* No actual flushing is required for the GTT write domain. Writes |
2792 | * to it immediately go to main memory as far as we know, so there's |
2789 | * to it immediately go to main memory as far as we know, so there's |
2793 | * no chipset flush. It also doesn't land in render cache. |
2790 | * no chipset flush. It also doesn't land in render cache. |
2794 | * |
2791 | * |
2795 | * However, we do have to enforce the order so that all writes through |
2792 | * However, we do have to enforce the order so that all writes through |
2796 | * the GTT land before any writes to the device, such as updates to |
2793 | * the GTT land before any writes to the device, such as updates to |
2797 | * the GATT itself. |
2794 | * the GATT itself. |
2798 | */ |
2795 | */ |
2799 | wmb(); |
2796 | wmb(); |
2800 | 2797 | ||
2801 | old_write_domain = obj->base.write_domain; |
2798 | old_write_domain = obj->base.write_domain; |
2802 | obj->base.write_domain = 0; |
2799 | obj->base.write_domain = 0; |
2803 | 2800 | ||
2804 | trace_i915_gem_object_change_domain(obj, |
2801 | trace_i915_gem_object_change_domain(obj, |
2805 | obj->base.read_domains, |
2802 | obj->base.read_domains, |
2806 | old_write_domain); |
2803 | old_write_domain); |
2807 | } |
2804 | } |
2808 | 2805 | ||
2809 | /** Flushes the CPU write domain for the object if it's dirty. */ |
2806 | /** Flushes the CPU write domain for the object if it's dirty. */ |
2810 | static void |
2807 | static void |
2811 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) |
2808 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) |
2812 | { |
2809 | { |
2813 | uint32_t old_write_domain; |
2810 | uint32_t old_write_domain; |
2814 | 2811 | ||
2815 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
2812 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
2816 | return; |
2813 | return; |
2817 | 2814 | ||
2818 | i915_gem_clflush_object(obj); |
2815 | i915_gem_clflush_object(obj); |
2819 | i915_gem_chipset_flush(obj->base.dev); |
2816 | i915_gem_chipset_flush(obj->base.dev); |
2820 | old_write_domain = obj->base.write_domain; |
2817 | old_write_domain = obj->base.write_domain; |
2821 | obj->base.write_domain = 0; |
2818 | obj->base.write_domain = 0; |
2822 | 2819 | ||
2823 | trace_i915_gem_object_change_domain(obj, |
2820 | trace_i915_gem_object_change_domain(obj, |
2824 | obj->base.read_domains, |
2821 | obj->base.read_domains, |
2825 | old_write_domain); |
2822 | old_write_domain); |
2826 | } |
2823 | } |
2827 | 2824 | ||
2828 | /** |
2825 | /** |
2829 | * Moves a single object to the GTT read, and possibly write domain. |
2826 | * Moves a single object to the GTT read, and possibly write domain. |
2830 | * |
2827 | * |
2831 | * This function returns when the move is complete, including waiting on |
2828 | * This function returns when the move is complete, including waiting on |
2832 | * flushes to occur. |
2829 | * flushes to occur. |
2833 | */ |
2830 | */ |
2834 | int |
2831 | int |
2835 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
2832 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
2836 | { |
2833 | { |
2837 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
2834 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
2838 | uint32_t old_write_domain, old_read_domains; |
2835 | uint32_t old_write_domain, old_read_domains; |
2839 | int ret; |
2836 | int ret; |
2840 | 2837 | ||
2841 | /* Not valid to be called on unbound objects. */ |
2838 | /* Not valid to be called on unbound objects. */ |
2842 | if (obj->gtt_space == NULL) |
2839 | if (obj->gtt_space == NULL) |
2843 | return -EINVAL; |
2840 | return -EINVAL; |
2844 | 2841 | ||
2845 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
2842 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
2846 | return 0; |
2843 | return 0; |
2847 | 2844 | ||
2848 | ret = i915_gem_object_wait_rendering(obj, !write); |
2845 | ret = i915_gem_object_wait_rendering(obj, !write); |
2849 | if (ret) |
2846 | if (ret) |
2850 | return ret; |
2847 | return ret; |
2851 | 2848 | ||
2852 | i915_gem_object_flush_cpu_write_domain(obj); |
2849 | i915_gem_object_flush_cpu_write_domain(obj); |
2853 | 2850 | ||
2854 | old_write_domain = obj->base.write_domain; |
2851 | old_write_domain = obj->base.write_domain; |
2855 | old_read_domains = obj->base.read_domains; |
2852 | old_read_domains = obj->base.read_domains; |
2856 | 2853 | ||
2857 | /* It should now be out of any other write domains, and we can update |
2854 | /* It should now be out of any other write domains, and we can update |
2858 | * the domain values for our changes. |
2855 | * the domain values for our changes. |
2859 | */ |
2856 | */ |
2860 | BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); |
2857 | BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); |
2861 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
2858 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
2862 | if (write) { |
2859 | if (write) { |
2863 | obj->base.read_domains = I915_GEM_DOMAIN_GTT; |
2860 | obj->base.read_domains = I915_GEM_DOMAIN_GTT; |
2864 | obj->base.write_domain = I915_GEM_DOMAIN_GTT; |
2861 | obj->base.write_domain = I915_GEM_DOMAIN_GTT; |
2865 | obj->dirty = 1; |
2862 | obj->dirty = 1; |
2866 | } |
2863 | } |
2867 | 2864 | ||
2868 | trace_i915_gem_object_change_domain(obj, |
2865 | trace_i915_gem_object_change_domain(obj, |
2869 | old_read_domains, |
2866 | old_read_domains, |
2870 | old_write_domain); |
2867 | old_write_domain); |
2871 | 2868 | ||
2872 | /* And bump the LRU for this access */ |
2869 | /* And bump the LRU for this access */ |
2873 | if (i915_gem_object_is_inactive(obj)) |
2870 | if (i915_gem_object_is_inactive(obj)) |
2874 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
2871 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
2875 | 2872 | ||
2876 | return 0; |
2873 | return 0; |
2877 | } |
2874 | } |
2878 | 2875 | ||
2879 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
2876 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
2880 | enum i915_cache_level cache_level) |
2877 | enum i915_cache_level cache_level) |
2881 | { |
2878 | { |
2882 | struct drm_device *dev = obj->base.dev; |
2879 | struct drm_device *dev = obj->base.dev; |
2883 | drm_i915_private_t *dev_priv = dev->dev_private; |
2880 | drm_i915_private_t *dev_priv = dev->dev_private; |
2884 | int ret; |
2881 | int ret; |
2885 | 2882 | ||
2886 | if (obj->cache_level == cache_level) |
2883 | if (obj->cache_level == cache_level) |
2887 | return 0; |
2884 | return 0; |
2888 | 2885 | ||
2889 | if (obj->pin_count) { |
2886 | if (obj->pin_count) { |
2890 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
2887 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
2891 | return -EBUSY; |
2888 | return -EBUSY; |
2892 | } |
2889 | } |
2893 | 2890 | ||
2894 | if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { |
2891 | if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { |
2895 | ret = i915_gem_object_unbind(obj); |
2892 | ret = i915_gem_object_unbind(obj); |
2896 | if (ret) |
2893 | if (ret) |
2897 | return ret; |
2894 | return ret; |
2898 | } |
2895 | } |
2899 | 2896 | ||
2900 | if (obj->gtt_space) { |
2897 | if (obj->gtt_space) { |
2901 | ret = i915_gem_object_finish_gpu(obj); |
2898 | ret = i915_gem_object_finish_gpu(obj); |
2902 | if (ret) |
2899 | if (ret) |
2903 | return ret; |
2900 | return ret; |
2904 | 2901 | ||
2905 | i915_gem_object_finish_gtt(obj); |
2902 | i915_gem_object_finish_gtt(obj); |
2906 | 2903 | ||
2907 | /* Before SandyBridge, you could not use tiling or fence |
2904 | /* Before SandyBridge, you could not use tiling or fence |
2908 | * registers with snooped memory, so relinquish any fences |
2905 | * registers with snooped memory, so relinquish any fences |
2909 | * currently pointing to our region in the aperture. |
2906 | * currently pointing to our region in the aperture. |
2910 | */ |
2907 | */ |
2911 | if (INTEL_INFO(dev)->gen < 6) { |
2908 | if (INTEL_INFO(dev)->gen < 6) { |
2912 | ret = i915_gem_object_put_fence(obj); |
2909 | ret = i915_gem_object_put_fence(obj); |
2913 | if (ret) |
2910 | if (ret) |
2914 | return ret; |
2911 | return ret; |
2915 | } |
2912 | } |
2916 | 2913 | ||
2917 | if (obj->has_global_gtt_mapping) |
2914 | if (obj->has_global_gtt_mapping) |
2918 | i915_gem_gtt_bind_object(obj, cache_level); |
2915 | i915_gem_gtt_bind_object(obj, cache_level); |
2919 | if (obj->has_aliasing_ppgtt_mapping) |
2916 | if (obj->has_aliasing_ppgtt_mapping) |
2920 | i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, |
2917 | i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, |
2921 | obj, cache_level); |
2918 | obj, cache_level); |
2922 | 2919 | ||
2923 | obj->gtt_space->color = cache_level; |
2920 | obj->gtt_space->color = cache_level; |
2924 | } |
2921 | } |
2925 | 2922 | ||
2926 | if (cache_level == I915_CACHE_NONE) { |
2923 | if (cache_level == I915_CACHE_NONE) { |
2927 | u32 old_read_domains, old_write_domain; |
2924 | u32 old_read_domains, old_write_domain; |
2928 | 2925 | ||
2929 | /* If we're coming from LLC cached, then we haven't |
2926 | /* If we're coming from LLC cached, then we haven't |
2930 | * actually been tracking whether the data is in the |
2927 | * actually been tracking whether the data is in the |
2931 | * CPU cache or not, since we only allow one bit set |
2928 | * CPU cache or not, since we only allow one bit set |
2932 | * in obj->write_domain and have been skipping the clflushes. |
2929 | * in obj->write_domain and have been skipping the clflushes. |
2933 | * Just set it to the CPU cache for now. |
2930 | * Just set it to the CPU cache for now. |
2934 | */ |
2931 | */ |
2935 | WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); |
2932 | WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); |
2936 | WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU); |
2933 | WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU); |
2937 | 2934 | ||
2938 | old_read_domains = obj->base.read_domains; |
2935 | old_read_domains = obj->base.read_domains; |
2939 | old_write_domain = obj->base.write_domain; |
2936 | old_write_domain = obj->base.write_domain; |
2940 | 2937 | ||
2941 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
2938 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
2942 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
2939 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
2943 | 2940 | ||
2944 | trace_i915_gem_object_change_domain(obj, |
2941 | trace_i915_gem_object_change_domain(obj, |
2945 | old_read_domains, |
2942 | old_read_domains, |
2946 | old_write_domain); |
2943 | old_write_domain); |
2947 | } |
2944 | } |
2948 | 2945 | ||
2949 | obj->cache_level = cache_level; |
2946 | obj->cache_level = cache_level; |
2950 | i915_gem_verify_gtt(dev); |
2947 | i915_gem_verify_gtt(dev); |
2951 | return 0; |
2948 | return 0; |
2952 | } |
2949 | } |
2953 | 2950 | ||
2954 | int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, |
2951 | int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, |
2955 | struct drm_file *file) |
2952 | struct drm_file *file) |
2956 | { |
2953 | { |
2957 | struct drm_i915_gem_caching *args = data; |
2954 | struct drm_i915_gem_caching *args = data; |
2958 | struct drm_i915_gem_object *obj; |
2955 | struct drm_i915_gem_object *obj; |
2959 | int ret; |
2956 | int ret; |
2960 | 2957 | ||
2961 | ret = i915_mutex_lock_interruptible(dev); |
2958 | ret = i915_mutex_lock_interruptible(dev); |
2962 | if (ret) |
2959 | if (ret) |
2963 | return ret; |
2960 | return ret; |
2964 | 2961 | ||
2965 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
2962 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
2966 | if (&obj->base == NULL) { |
2963 | if (&obj->base == NULL) { |
2967 | ret = -ENOENT; |
2964 | ret = -ENOENT; |
2968 | goto unlock; |
2965 | goto unlock; |
2969 | } |
2966 | } |
2970 | 2967 | ||
2971 | args->caching = obj->cache_level != I915_CACHE_NONE; |
2968 | args->caching = obj->cache_level != I915_CACHE_NONE; |
2972 | 2969 | ||
2973 | drm_gem_object_unreference(&obj->base); |
2970 | drm_gem_object_unreference(&obj->base); |
2974 | unlock: |
2971 | unlock: |
2975 | mutex_unlock(&dev->struct_mutex); |
2972 | mutex_unlock(&dev->struct_mutex); |
2976 | return ret; |
2973 | return ret; |
2977 | } |
2974 | } |
2978 | 2975 | ||
2979 | int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, |
2976 | int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, |
2980 | struct drm_file *file) |
2977 | struct drm_file *file) |
2981 | { |
2978 | { |
2982 | struct drm_i915_gem_caching *args = data; |
2979 | struct drm_i915_gem_caching *args = data; |
2983 | struct drm_i915_gem_object *obj; |
2980 | struct drm_i915_gem_object *obj; |
2984 | enum i915_cache_level level; |
2981 | enum i915_cache_level level; |
2985 | int ret; |
2982 | int ret; |
2986 | 2983 | ||
2987 | switch (args->caching) { |
2984 | switch (args->caching) { |
2988 | case I915_CACHING_NONE: |
2985 | case I915_CACHING_NONE: |
2989 | level = I915_CACHE_NONE; |
2986 | level = I915_CACHE_NONE; |
2990 | break; |
2987 | break; |
2991 | case I915_CACHING_CACHED: |
2988 | case I915_CACHING_CACHED: |
2992 | level = I915_CACHE_LLC; |
2989 | level = I915_CACHE_LLC; |
2993 | break; |
2990 | break; |
2994 | default: |
2991 | default: |
2995 | return -EINVAL; |
2992 | return -EINVAL; |
2996 | } |
2993 | } |
2997 | 2994 | ||
2998 | ret = i915_mutex_lock_interruptible(dev); |
2995 | ret = i915_mutex_lock_interruptible(dev); |
2999 | if (ret) |
2996 | if (ret) |
3000 | return ret; |
2997 | return ret; |
3001 | 2998 | ||
3002 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
2999 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3003 | if (&obj->base == NULL) { |
3000 | if (&obj->base == NULL) { |
3004 | ret = -ENOENT; |
3001 | ret = -ENOENT; |
3005 | goto unlock; |
3002 | goto unlock; |
3006 | } |
3003 | } |
3007 | 3004 | ||
3008 | ret = i915_gem_object_set_cache_level(obj, level); |
3005 | ret = i915_gem_object_set_cache_level(obj, level); |
3009 | 3006 | ||
3010 | drm_gem_object_unreference(&obj->base); |
3007 | drm_gem_object_unreference(&obj->base); |
3011 | unlock: |
3008 | unlock: |
3012 | mutex_unlock(&dev->struct_mutex); |
3009 | mutex_unlock(&dev->struct_mutex); |
3013 | return ret; |
3010 | return ret; |
3014 | } |
3011 | } |
3015 | 3012 | ||
3016 | /* |
3013 | /* |
3017 | * Prepare buffer for display plane (scanout, cursors, etc). |
3014 | * Prepare buffer for display plane (scanout, cursors, etc). |
3018 | * Can be called from an uninterruptible phase (modesetting) and allows |
3015 | * Can be called from an uninterruptible phase (modesetting) and allows |
3019 | * any flushes to be pipelined (for pageflips). |
3016 | * any flushes to be pipelined (for pageflips). |
3020 | */ |
3017 | */ |
3021 | int |
3018 | int |
3022 | i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, |
3019 | i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, |
3023 | u32 alignment, |
3020 | u32 alignment, |
3024 | struct intel_ring_buffer *pipelined) |
3021 | struct intel_ring_buffer *pipelined) |
3025 | { |
3022 | { |
3026 | u32 old_read_domains, old_write_domain; |
3023 | u32 old_read_domains, old_write_domain; |
3027 | int ret; |
3024 | int ret; |
3028 | 3025 | ||
3029 | if (pipelined != obj->ring) { |
3026 | if (pipelined != obj->ring) { |
3030 | ret = i915_gem_object_sync(obj, pipelined); |
3027 | ret = i915_gem_object_sync(obj, pipelined); |
3031 | if (ret) |
3028 | if (ret) |
3032 | return ret; |
3029 | return ret; |
3033 | } |
3030 | } |
3034 | 3031 | ||
3035 | /* The display engine is not coherent with the LLC cache on gen6. As |
3032 | /* The display engine is not coherent with the LLC cache on gen6. As |
3036 | * a result, we make sure that the pinning that is about to occur is |
3033 | * a result, we make sure that the pinning that is about to occur is |
3037 | * done with uncached PTEs. This is lowest common denominator for all |
3034 | * done with uncached PTEs. This is lowest common denominator for all |
3038 | * chipsets. |
3035 | * chipsets. |
3039 | * |
3036 | * |
3040 | * However for gen6+, we could do better by using the GFDT bit instead |
3037 | * However for gen6+, we could do better by using the GFDT bit instead |
3041 | * of uncaching, which would allow us to flush all the LLC-cached data |
3038 | * of uncaching, which would allow us to flush all the LLC-cached data |
3042 | * with that bit in the PTE to main memory with just one PIPE_CONTROL. |
3039 | * with that bit in the PTE to main memory with just one PIPE_CONTROL. |
3043 | */ |
3040 | */ |
3044 | ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); |
3041 | ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); |
3045 | if (ret) |
3042 | if (ret) |
3046 | return ret; |
3043 | return ret; |
3047 | 3044 | ||
3048 | /* As the user may map the buffer once pinned in the display plane |
3045 | /* As the user may map the buffer once pinned in the display plane |
3049 | * (e.g. libkms for the bootup splash), we have to ensure that we |
3046 | * (e.g. libkms for the bootup splash), we have to ensure that we |
3050 | * always use map_and_fenceable for all scanout buffers. |
3047 | * always use map_and_fenceable for all scanout buffers. |
3051 | */ |
3048 | */ |
3052 | ret = i915_gem_object_pin(obj, alignment, true, false); |
3049 | ret = i915_gem_object_pin(obj, alignment, true, false); |
3053 | if (ret) |
3050 | if (ret) |
3054 | return ret; |
3051 | return ret; |
3055 | 3052 | ||
3056 | i915_gem_object_flush_cpu_write_domain(obj); |
3053 | i915_gem_object_flush_cpu_write_domain(obj); |
3057 | 3054 | ||
3058 | old_write_domain = obj->base.write_domain; |
3055 | old_write_domain = obj->base.write_domain; |
3059 | old_read_domains = obj->base.read_domains; |
3056 | old_read_domains = obj->base.read_domains; |
3060 | 3057 | ||
3061 | /* It should now be out of any other write domains, and we can update |
3058 | /* It should now be out of any other write domains, and we can update |
3062 | * the domain values for our changes. |
3059 | * the domain values for our changes. |
3063 | */ |
3060 | */ |
3064 | obj->base.write_domain = 0; |
3061 | obj->base.write_domain = 0; |
3065 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
3062 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
3066 | 3063 | ||
3067 | trace_i915_gem_object_change_domain(obj, |
3064 | trace_i915_gem_object_change_domain(obj, |
3068 | old_read_domains, |
3065 | old_read_domains, |
3069 | old_write_domain); |
3066 | old_write_domain); |
3070 | 3067 | ||
3071 | return 0; |
3068 | return 0; |
3072 | } |
3069 | } |
3073 | 3070 | ||
3074 | int |
3071 | int |
3075 | i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) |
3072 | i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) |
3076 | { |
3073 | { |
3077 | int ret; |
3074 | int ret; |
3078 | 3075 | ||
3079 | if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) |
3076 | if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) |
3080 | return 0; |
3077 | return 0; |
3081 | 3078 | ||
3082 | ret = i915_gem_object_wait_rendering(obj, false); |
3079 | ret = i915_gem_object_wait_rendering(obj, false); |
3083 | if (ret) |
3080 | if (ret) |
3084 | return ret; |
3081 | return ret; |
3085 | 3082 | ||
3086 | /* Ensure that we invalidate the GPU's caches and TLBs. */ |
3083 | /* Ensure that we invalidate the GPU's caches and TLBs. */ |
3087 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
3084 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
3088 | return 0; |
3085 | return 0; |
3089 | } |
3086 | } |
3090 | 3087 | ||
3091 | /** |
3088 | /** |
3092 | * Moves a single object to the CPU read, and possibly write domain. |
3089 | * Moves a single object to the CPU read, and possibly write domain. |
3093 | * |
3090 | * |
3094 | * This function returns when the move is complete, including waiting on |
3091 | * This function returns when the move is complete, including waiting on |
3095 | * flushes to occur. |
3092 | * flushes to occur. |
3096 | */ |
3093 | */ |
3097 | int |
3094 | int |
3098 | i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) |
3095 | i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) |
3099 | { |
3096 | { |
3100 | uint32_t old_write_domain, old_read_domains; |
3097 | uint32_t old_write_domain, old_read_domains; |
3101 | int ret; |
3098 | int ret; |
3102 | 3099 | ||
3103 | if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) |
3100 | if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) |
3104 | return 0; |
3101 | return 0; |
3105 | 3102 | ||
3106 | ret = i915_gem_object_wait_rendering(obj, !write); |
3103 | ret = i915_gem_object_wait_rendering(obj, !write); |
3107 | if (ret) |
3104 | if (ret) |
3108 | return ret; |
3105 | return ret; |
3109 | 3106 | ||
3110 | i915_gem_object_flush_gtt_write_domain(obj); |
3107 | i915_gem_object_flush_gtt_write_domain(obj); |
3111 | 3108 | ||
3112 | old_write_domain = obj->base.write_domain; |
3109 | old_write_domain = obj->base.write_domain; |
3113 | old_read_domains = obj->base.read_domains; |
3110 | old_read_domains = obj->base.read_domains; |
3114 | 3111 | ||
3115 | /* Flush the CPU cache if it's still invalid. */ |
3112 | /* Flush the CPU cache if it's still invalid. */ |
3116 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { |
3113 | if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { |
3117 | i915_gem_clflush_object(obj); |
3114 | i915_gem_clflush_object(obj); |
3118 | 3115 | ||
3119 | obj->base.read_domains |= I915_GEM_DOMAIN_CPU; |
3116 | obj->base.read_domains |= I915_GEM_DOMAIN_CPU; |
3120 | } |
3117 | } |
3121 | 3118 | ||
3122 | /* It should now be out of any other write domains, and we can update |
3119 | /* It should now be out of any other write domains, and we can update |
3123 | * the domain values for our changes. |
3120 | * the domain values for our changes. |
3124 | */ |
3121 | */ |
3125 | BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); |
3122 | BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); |
3126 | 3123 | ||
3127 | /* If we're writing through the CPU, then the GPU read domains will |
3124 | /* If we're writing through the CPU, then the GPU read domains will |
3128 | * need to be invalidated at next use. |
3125 | * need to be invalidated at next use. |
3129 | */ |
3126 | */ |
3130 | if (write) { |
3127 | if (write) { |
3131 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3128 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3132 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3129 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3133 | } |
3130 | } |
3134 | 3131 | ||
3135 | trace_i915_gem_object_change_domain(obj, |
3132 | trace_i915_gem_object_change_domain(obj, |
3136 | old_read_domains, |
3133 | old_read_domains, |
3137 | old_write_domain); |
3134 | old_write_domain); |
3138 | 3135 | ||
3139 | return 0; |
3136 | return 0; |
3140 | } |
3137 | } |
3141 | 3138 | ||
3142 | /* Throttle our rendering by waiting until the ring has completed our requests |
3139 | /* Throttle our rendering by waiting until the ring has completed our requests |
3143 | * emitted over 20 msec ago. |
3140 | * emitted over 20 msec ago. |
3144 | * |
3141 | * |
3145 | * Note that if we were to use the current jiffies each time around the loop, |
3142 | * Note that if we were to use the current jiffies each time around the loop, |
3146 | * we wouldn't escape the function with any frames outstanding if the time to |
3143 | * we wouldn't escape the function with any frames outstanding if the time to |
3147 | * render a frame was over 20ms. |
3144 | * render a frame was over 20ms. |
3148 | * |
3145 | * |
3149 | * This should get us reasonable parallelism between CPU and GPU but also |
3146 | * This should get us reasonable parallelism between CPU and GPU but also |
3150 | * relatively low latency when blocking on a particular request to finish. |
3147 | * relatively low latency when blocking on a particular request to finish. |
3151 | */ |
3148 | */ |
3152 | static int |
3149 | static int |
3153 | i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) |
3150 | i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) |
3154 | { |
3151 | { |
3155 | struct drm_i915_private *dev_priv = dev->dev_private; |
3152 | struct drm_i915_private *dev_priv = dev->dev_private; |
3156 | struct drm_i915_file_private *file_priv = file->driver_priv; |
3153 | struct drm_i915_file_private *file_priv = file->driver_priv; |
3157 | unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20); |
3154 | unsigned long recent_enough = GetTimerTicks() - msecs_to_jiffies(20); |
3158 | struct drm_i915_gem_request *request; |
3155 | struct drm_i915_gem_request *request; |
3159 | struct intel_ring_buffer *ring = NULL; |
3156 | struct intel_ring_buffer *ring = NULL; |
3160 | u32 seqno = 0; |
3157 | u32 seqno = 0; |
3161 | int ret; |
3158 | int ret; |
3162 | 3159 | ||
3163 | if (atomic_read(&dev_priv->mm.wedged)) |
3160 | if (atomic_read(&dev_priv->mm.wedged)) |
3164 | return -EIO; |
3161 | return -EIO; |
3165 | 3162 | ||
3166 | spin_lock(&file_priv->mm.lock); |
3163 | spin_lock(&file_priv->mm.lock); |
3167 | list_for_each_entry(request, &file_priv->mm.request_list, client_list) { |
3164 | list_for_each_entry(request, &file_priv->mm.request_list, client_list) { |
3168 | if (time_after_eq(request->emitted_jiffies, recent_enough)) |
3165 | if (time_after_eq(request->emitted_jiffies, recent_enough)) |
3169 | break; |
3166 | break; |
3170 | 3167 | ||
3171 | ring = request->ring; |
3168 | ring = request->ring; |
3172 | seqno = request->seqno; |
3169 | seqno = request->seqno; |
3173 | } |
3170 | } |
3174 | spin_unlock(&file_priv->mm.lock); |
3171 | spin_unlock(&file_priv->mm.lock); |
3175 | 3172 | ||
3176 | if (seqno == 0) |
3173 | if (seqno == 0) |
3177 | return 0; |
3174 | return 0; |
3178 | 3175 | ||
3179 | ret = __wait_seqno(ring, seqno, true, NULL); |
3176 | ret = __wait_seqno(ring, seqno, true, NULL); |
3180 | if (ret == 0) |
3177 | if (ret == 0) |
3181 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); |
3178 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); |
3182 | 3179 | ||
3183 | return ret; |
3180 | return ret; |
3184 | } |
3181 | } |
3185 | 3182 | ||
3186 | int |
3183 | int |
3187 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
3184 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
3188 | uint32_t alignment, |
3185 | uint32_t alignment, |
3189 | bool map_and_fenceable, |
3186 | bool map_and_fenceable, |
3190 | bool nonblocking) |
3187 | bool nonblocking) |
3191 | { |
3188 | { |
3192 | int ret; |
3189 | int ret; |
3193 | 3190 | ||
3194 | if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) |
3191 | if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) |
3195 | return -EBUSY; |
3192 | return -EBUSY; |
3196 | 3193 | ||
3197 | if (obj->gtt_space != NULL) { |
3194 | if (obj->gtt_space != NULL) { |
3198 | if ((alignment && obj->gtt_offset & (alignment - 1)) || |
3195 | if ((alignment && obj->gtt_offset & (alignment - 1)) || |
3199 | (map_and_fenceable && !obj->map_and_fenceable)) { |
3196 | (map_and_fenceable && !obj->map_and_fenceable)) { |
3200 | WARN(obj->pin_count, |
3197 | WARN(obj->pin_count, |
3201 | "bo is already pinned with incorrect alignment:" |
3198 | "bo is already pinned with incorrect alignment:" |
3202 | " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," |
3199 | " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," |
3203 | " obj->map_and_fenceable=%d\n", |
3200 | " obj->map_and_fenceable=%d\n", |
3204 | obj->gtt_offset, alignment, |
3201 | obj->gtt_offset, alignment, |
3205 | map_and_fenceable, |
3202 | map_and_fenceable, |
3206 | obj->map_and_fenceable); |
3203 | obj->map_and_fenceable); |
3207 | ret = i915_gem_object_unbind(obj); |
3204 | ret = i915_gem_object_unbind(obj); |
3208 | if (ret) |
3205 | if (ret) |
3209 | return ret; |
3206 | return ret; |
3210 | } |
3207 | } |
3211 | } |
3208 | } |
3212 | 3209 | ||
3213 | if (obj->gtt_space == NULL) { |
3210 | if (obj->gtt_space == NULL) { |
3214 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
3211 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
3215 | 3212 | ||
3216 | ret = i915_gem_object_bind_to_gtt(obj, alignment, |
3213 | ret = i915_gem_object_bind_to_gtt(obj, alignment, |
3217 | map_and_fenceable, |
3214 | map_and_fenceable, |
3218 | nonblocking); |
3215 | nonblocking); |
3219 | if (ret) |
3216 | if (ret) |
3220 | return ret; |
3217 | return ret; |
3221 | 3218 | ||
3222 | if (!dev_priv->mm.aliasing_ppgtt) |
3219 | if (!dev_priv->mm.aliasing_ppgtt) |
3223 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
3220 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
3224 | } |
3221 | } |
3225 | 3222 | ||
3226 | if (!obj->has_global_gtt_mapping && map_and_fenceable) |
3223 | if (!obj->has_global_gtt_mapping && map_and_fenceable) |
3227 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
3224 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
3228 | 3225 | ||
3229 | obj->pin_count++; |
3226 | obj->pin_count++; |
3230 | obj->pin_mappable |= map_and_fenceable; |
3227 | obj->pin_mappable |= map_and_fenceable; |
3231 | 3228 | ||
3232 | return 0; |
3229 | return 0; |
3233 | } |
3230 | } |
3234 | 3231 | ||
3235 | void |
3232 | void |
3236 | i915_gem_object_unpin(struct drm_i915_gem_object *obj) |
3233 | i915_gem_object_unpin(struct drm_i915_gem_object *obj) |
3237 | { |
3234 | { |
3238 | BUG_ON(obj->pin_count == 0); |
3235 | BUG_ON(obj->pin_count == 0); |
3239 | BUG_ON(obj->gtt_space == NULL); |
3236 | BUG_ON(obj->gtt_space == NULL); |
3240 | 3237 | ||
3241 | if (--obj->pin_count == 0) |
3238 | if (--obj->pin_count == 0) |
3242 | obj->pin_mappable = false; |
3239 | obj->pin_mappable = false; |
3243 | } |
3240 | } |
3244 | 3241 | ||
3245 | int |
3242 | int |
3246 | i915_gem_pin_ioctl(struct drm_device *dev, void *data, |
3243 | i915_gem_pin_ioctl(struct drm_device *dev, void *data, |
3247 | struct drm_file *file) |
3244 | struct drm_file *file) |
3248 | { |
3245 | { |
3249 | struct drm_i915_gem_pin *args = data; |
3246 | struct drm_i915_gem_pin *args = data; |
3250 | struct drm_i915_gem_object *obj; |
3247 | struct drm_i915_gem_object *obj; |
3251 | int ret; |
3248 | int ret; |
3252 | 3249 | ||
3253 | ret = i915_mutex_lock_interruptible(dev); |
3250 | ret = i915_mutex_lock_interruptible(dev); |
3254 | if (ret) |
3251 | if (ret) |
3255 | return ret; |
3252 | return ret; |
3256 | 3253 | ||
3257 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3254 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3258 | if (&obj->base == NULL) { |
3255 | if (&obj->base == NULL) { |
3259 | ret = -ENOENT; |
3256 | ret = -ENOENT; |
3260 | goto unlock; |
3257 | goto unlock; |
3261 | } |
3258 | } |
3262 | 3259 | ||
3263 | if (obj->madv != I915_MADV_WILLNEED) { |
3260 | if (obj->madv != I915_MADV_WILLNEED) { |
3264 | DRM_ERROR("Attempting to pin a purgeable buffer\n"); |
3261 | DRM_ERROR("Attempting to pin a purgeable buffer\n"); |
3265 | ret = -EINVAL; |
3262 | ret = -EINVAL; |
3266 | goto out; |
3263 | goto out; |
3267 | } |
3264 | } |
3268 | 3265 | ||
3269 | if (obj->pin_filp != NULL && obj->pin_filp != file) { |
3266 | if (obj->pin_filp != NULL && obj->pin_filp != file) { |
3270 | DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", |
3267 | DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", |
3271 | args->handle); |
3268 | args->handle); |
3272 | ret = -EINVAL; |
3269 | ret = -EINVAL; |
3273 | goto out; |
3270 | goto out; |
3274 | } |
3271 | } |
3275 | 3272 | ||
3276 | if (obj->user_pin_count == 0) { |
3273 | if (obj->user_pin_count == 0) { |
3277 | ret = i915_gem_object_pin(obj, args->alignment, true, false); |
3274 | ret = i915_gem_object_pin(obj, args->alignment, true, false); |
3278 | if (ret) |
3275 | if (ret) |
3279 | goto out; |
3276 | goto out; |
3280 | } |
3277 | } |
3281 | 3278 | ||
3282 | obj->user_pin_count++; |
3279 | obj->user_pin_count++; |
3283 | obj->pin_filp = file; |
3280 | obj->pin_filp = file; |
3284 | 3281 | ||
3285 | /* XXX - flush the CPU caches for pinned objects |
3282 | /* XXX - flush the CPU caches for pinned objects |
3286 | * as the X server doesn't manage domains yet |
3283 | * as the X server doesn't manage domains yet |
3287 | */ |
3284 | */ |
3288 | i915_gem_object_flush_cpu_write_domain(obj); |
3285 | i915_gem_object_flush_cpu_write_domain(obj); |
3289 | args->offset = obj->gtt_offset; |
3286 | args->offset = obj->gtt_offset; |
3290 | out: |
3287 | out: |
3291 | drm_gem_object_unreference(&obj->base); |
3288 | drm_gem_object_unreference(&obj->base); |
3292 | unlock: |
3289 | unlock: |
3293 | mutex_unlock(&dev->struct_mutex); |
3290 | mutex_unlock(&dev->struct_mutex); |
3294 | return ret; |
3291 | return ret; |
3295 | } |
3292 | } |
3296 | 3293 | ||
3297 | #if 0 |
3294 | #if 0 |
3298 | 3295 | ||
3299 | int |
3296 | int |
3300 | i915_gem_unpin_ioctl(struct drm_device *dev, void *data, |
3297 | i915_gem_unpin_ioctl(struct drm_device *dev, void *data, |
3301 | struct drm_file *file) |
3298 | struct drm_file *file) |
3302 | { |
3299 | { |
3303 | struct drm_i915_gem_pin *args = data; |
3300 | struct drm_i915_gem_pin *args = data; |
3304 | struct drm_i915_gem_object *obj; |
3301 | struct drm_i915_gem_object *obj; |
3305 | int ret; |
3302 | int ret; |
3306 | 3303 | ||
3307 | ret = i915_mutex_lock_interruptible(dev); |
3304 | ret = i915_mutex_lock_interruptible(dev); |
3308 | if (ret) |
3305 | if (ret) |
3309 | return ret; |
3306 | return ret; |
3310 | 3307 | ||
3311 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3308 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3312 | if (&obj->base == NULL) { |
3309 | if (&obj->base == NULL) { |
3313 | ret = -ENOENT; |
3310 | ret = -ENOENT; |
3314 | goto unlock; |
3311 | goto unlock; |
3315 | } |
3312 | } |
3316 | 3313 | ||
3317 | if (obj->pin_filp != file) { |
3314 | if (obj->pin_filp != file) { |
3318 | DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", |
3315 | DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", |
3319 | args->handle); |
3316 | args->handle); |
3320 | ret = -EINVAL; |
3317 | ret = -EINVAL; |
3321 | goto out; |
3318 | goto out; |
3322 | } |
3319 | } |
3323 | obj->user_pin_count--; |
3320 | obj->user_pin_count--; |
3324 | if (obj->user_pin_count == 0) { |
3321 | if (obj->user_pin_count == 0) { |
3325 | obj->pin_filp = NULL; |
3322 | obj->pin_filp = NULL; |
3326 | i915_gem_object_unpin(obj); |
3323 | i915_gem_object_unpin(obj); |
3327 | } |
3324 | } |
3328 | 3325 | ||
3329 | out: |
3326 | out: |
3330 | drm_gem_object_unreference(&obj->base); |
3327 | drm_gem_object_unreference(&obj->base); |
3331 | unlock: |
3328 | unlock: |
3332 | mutex_unlock(&dev->struct_mutex); |
3329 | mutex_unlock(&dev->struct_mutex); |
3333 | return ret; |
3330 | return ret; |
3334 | } |
3331 | } |
3335 | 3332 | ||
3336 | #endif |
3333 | #endif |
3337 | 3334 | ||
3338 | int |
3335 | int |
3339 | i915_gem_busy_ioctl(struct drm_device *dev, void *data, |
3336 | i915_gem_busy_ioctl(struct drm_device *dev, void *data, |
3340 | struct drm_file *file) |
3337 | struct drm_file *file) |
3341 | { |
3338 | { |
3342 | struct drm_i915_gem_busy *args = data; |
3339 | struct drm_i915_gem_busy *args = data; |
3343 | struct drm_i915_gem_object *obj; |
3340 | struct drm_i915_gem_object *obj; |
3344 | int ret; |
3341 | int ret; |
3345 | 3342 | ||
3346 | ret = i915_mutex_lock_interruptible(dev); |
3343 | ret = i915_mutex_lock_interruptible(dev); |
3347 | if (ret) |
3344 | if (ret) |
3348 | return ret; |
3345 | return ret; |
3349 | 3346 | ||
3350 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3347 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
3351 | if (&obj->base == NULL) { |
3348 | if (&obj->base == NULL) { |
3352 | ret = -ENOENT; |
3349 | ret = -ENOENT; |
3353 | goto unlock; |
3350 | goto unlock; |
3354 | } |
3351 | } |
3355 | 3352 | ||
3356 | /* Count all active objects as busy, even if they are currently not used |
3353 | /* Count all active objects as busy, even if they are currently not used |
3357 | * by the gpu. Users of this interface expect objects to eventually |
3354 | * by the gpu. Users of this interface expect objects to eventually |
3358 | * become non-busy without any further actions, therefore emit any |
3355 | * become non-busy without any further actions, therefore emit any |
3359 | * necessary flushes here. |
3356 | * necessary flushes here. |
3360 | */ |
3357 | */ |
3361 | ret = i915_gem_object_flush_active(obj); |
3358 | ret = i915_gem_object_flush_active(obj); |
3362 | 3359 | ||
3363 | args->busy = obj->active; |
3360 | args->busy = obj->active; |
3364 | if (obj->ring) { |
3361 | if (obj->ring) { |
3365 | BUILD_BUG_ON(I915_NUM_RINGS > 16); |
3362 | BUILD_BUG_ON(I915_NUM_RINGS > 16); |
3366 | args->busy |= intel_ring_flag(obj->ring) << 16; |
3363 | args->busy |= intel_ring_flag(obj->ring) << 16; |
3367 | } |
3364 | } |
3368 | 3365 | ||
3369 | drm_gem_object_unreference(&obj->base); |
3366 | drm_gem_object_unreference(&obj->base); |
3370 | unlock: |
3367 | unlock: |
3371 | mutex_unlock(&dev->struct_mutex); |
3368 | mutex_unlock(&dev->struct_mutex); |
3372 | return ret; |
3369 | return ret; |
3373 | } |
3370 | } |
3374 | 3371 | ||
3375 | int |
3372 | int |
3376 | i915_gem_throttle_ioctl(struct drm_device *dev, void *data, |
3373 | i915_gem_throttle_ioctl(struct drm_device *dev, void *data, |
3377 | struct drm_file *file_priv) |
3374 | struct drm_file *file_priv) |
3378 | { |
3375 | { |
3379 | return i915_gem_ring_throttle(dev, file_priv); |
3376 | return i915_gem_ring_throttle(dev, file_priv); |
3380 | } |
3377 | } |
3381 | 3378 | ||
3382 | #if 0 |
3379 | #if 0 |
3383 | 3380 | ||
3384 | int |
3381 | int |
3385 | i915_gem_madvise_ioctl(struct drm_device *dev, void *data, |
3382 | i915_gem_madvise_ioctl(struct drm_device *dev, void *data, |
3386 | struct drm_file *file_priv) |
3383 | struct drm_file *file_priv) |
3387 | { |
3384 | { |
3388 | struct drm_i915_gem_madvise *args = data; |
3385 | struct drm_i915_gem_madvise *args = data; |
3389 | struct drm_i915_gem_object *obj; |
3386 | struct drm_i915_gem_object *obj; |
3390 | int ret; |
3387 | int ret; |
3391 | 3388 | ||
3392 | switch (args->madv) { |
3389 | switch (args->madv) { |
3393 | case I915_MADV_DONTNEED: |
3390 | case I915_MADV_DONTNEED: |
3394 | case I915_MADV_WILLNEED: |
3391 | case I915_MADV_WILLNEED: |
3395 | break; |
3392 | break; |
3396 | default: |
3393 | default: |
3397 | return -EINVAL; |
3394 | return -EINVAL; |
3398 | } |
3395 | } |
3399 | 3396 | ||
3400 | ret = i915_mutex_lock_interruptible(dev); |
3397 | ret = i915_mutex_lock_interruptible(dev); |
3401 | if (ret) |
3398 | if (ret) |
3402 | return ret; |
3399 | return ret; |
3403 | 3400 | ||
3404 | obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); |
3401 | obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); |
3405 | if (&obj->base == NULL) { |
3402 | if (&obj->base == NULL) { |
3406 | ret = -ENOENT; |
3403 | ret = -ENOENT; |
3407 | goto unlock; |
3404 | goto unlock; |
3408 | } |
3405 | } |
3409 | 3406 | ||
3410 | if (obj->pin_count) { |
3407 | if (obj->pin_count) { |
3411 | ret = -EINVAL; |
3408 | ret = -EINVAL; |
3412 | goto out; |
3409 | goto out; |
3413 | } |
3410 | } |
3414 | 3411 | ||
3415 | if (obj->madv != __I915_MADV_PURGED) |
3412 | if (obj->madv != __I915_MADV_PURGED) |
3416 | obj->madv = args->madv; |
3413 | obj->madv = args->madv; |
3417 | 3414 | ||
3418 | /* if the object is no longer attached, discard its backing storage */ |
3415 | /* if the object is no longer attached, discard its backing storage */ |
3419 | if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) |
3416 | if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) |
3420 | i915_gem_object_truncate(obj); |
3417 | i915_gem_object_truncate(obj); |
3421 | 3418 | ||
3422 | args->retained = obj->madv != __I915_MADV_PURGED; |
3419 | args->retained = obj->madv != __I915_MADV_PURGED; |
3423 | 3420 | ||
3424 | out: |
3421 | out: |
3425 | drm_gem_object_unreference(&obj->base); |
3422 | drm_gem_object_unreference(&obj->base); |
3426 | unlock: |
3423 | unlock: |
3427 | mutex_unlock(&dev->struct_mutex); |
3424 | mutex_unlock(&dev->struct_mutex); |
3428 | return ret; |
3425 | return ret; |
3429 | } |
3426 | } |
3430 | #endif |
3427 | #endif |
3431 | 3428 | ||
3432 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
3429 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
3433 | const struct drm_i915_gem_object_ops *ops) |
3430 | const struct drm_i915_gem_object_ops *ops) |
3434 | { |
3431 | { |
3435 | INIT_LIST_HEAD(&obj->mm_list); |
3432 | INIT_LIST_HEAD(&obj->mm_list); |
3436 | INIT_LIST_HEAD(&obj->gtt_list); |
3433 | INIT_LIST_HEAD(&obj->gtt_list); |
3437 | INIT_LIST_HEAD(&obj->ring_list); |
3434 | INIT_LIST_HEAD(&obj->ring_list); |
3438 | INIT_LIST_HEAD(&obj->exec_list); |
3435 | INIT_LIST_HEAD(&obj->exec_list); |
3439 | 3436 | ||
3440 | obj->ops = ops; |
3437 | obj->ops = ops; |
3441 | 3438 | ||
3442 | obj->fence_reg = I915_FENCE_REG_NONE; |
3439 | obj->fence_reg = I915_FENCE_REG_NONE; |
3443 | obj->madv = I915_MADV_WILLNEED; |
3440 | obj->madv = I915_MADV_WILLNEED; |
3444 | /* Avoid an unnecessary call to unbind on the first bind. */ |
3441 | /* Avoid an unnecessary call to unbind on the first bind. */ |
3445 | obj->map_and_fenceable = true; |
3442 | obj->map_and_fenceable = true; |
3446 | 3443 | ||
3447 | i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); |
3444 | i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); |
3448 | } |
3445 | } |
3449 | 3446 | ||
3450 | static const struct drm_i915_gem_object_ops i915_gem_object_ops = { |
3447 | static const struct drm_i915_gem_object_ops i915_gem_object_ops = { |
3451 | .get_pages = i915_gem_object_get_pages_gtt, |
3448 | .get_pages = i915_gem_object_get_pages_gtt, |
3452 | .put_pages = i915_gem_object_put_pages_gtt, |
3449 | .put_pages = i915_gem_object_put_pages_gtt, |
3453 | }; |
3450 | }; |
3454 | 3451 | ||
3455 | struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, |
3452 | struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, |
3456 | size_t size) |
3453 | size_t size) |
3457 | { |
3454 | { |
3458 | struct drm_i915_gem_object *obj; |
3455 | struct drm_i915_gem_object *obj; |
3459 | struct address_space *mapping; |
3456 | struct address_space *mapping; |
3460 | u32 mask; |
3457 | u32 mask; |
3461 | 3458 | ||
3462 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); |
3459 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); |
3463 | if (obj == NULL) |
3460 | if (obj == NULL) |
3464 | return NULL; |
3461 | return NULL; |
3465 | 3462 | ||
3466 | if (drm_gem_object_init(dev, &obj->base, size) != 0) { |
3463 | if (drm_gem_object_init(dev, &obj->base, size) != 0) { |
3467 | kfree(obj); |
3464 | kfree(obj); |
3468 | return NULL; |
3465 | return NULL; |
3469 | } |
3466 | } |
3470 | 3467 | ||
3471 | 3468 | ||
3472 | i915_gem_object_init(obj, &i915_gem_object_ops); |
3469 | i915_gem_object_init(obj, &i915_gem_object_ops); |
3473 | 3470 | ||
3474 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3471 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3475 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3472 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3476 | 3473 | ||
3477 | if (HAS_LLC(dev)) { |
3474 | if (HAS_LLC(dev)) { |
3478 | /* On some devices, we can have the GPU use the LLC (the CPU |
3475 | /* On some devices, we can have the GPU use the LLC (the CPU |
3479 | * cache) for about a 10% performance improvement |
3476 | * cache) for about a 10% performance improvement |
3480 | * compared to uncached. Graphics requests other than |
3477 | * compared to uncached. Graphics requests other than |
3481 | * display scanout are coherent with the CPU in |
3478 | * display scanout are coherent with the CPU in |
3482 | * accessing this cache. This means in this mode we |
3479 | * accessing this cache. This means in this mode we |
3483 | * don't need to clflush on the CPU side, and on the |
3480 | * don't need to clflush on the CPU side, and on the |
3484 | * GPU side we only need to flush internal caches to |
3481 | * GPU side we only need to flush internal caches to |
3485 | * get data visible to the CPU. |
3482 | * get data visible to the CPU. |
3486 | * |
3483 | * |
3487 | * However, we maintain the display planes as UC, and so |
3484 | * However, we maintain the display planes as UC, and so |
3488 | * need to rebind when first used as such. |
3485 | * need to rebind when first used as such. |
3489 | */ |
3486 | */ |
3490 | obj->cache_level = I915_CACHE_LLC; |
3487 | obj->cache_level = I915_CACHE_LLC; |
3491 | } else |
3488 | } else |
3492 | obj->cache_level = I915_CACHE_NONE; |
3489 | obj->cache_level = I915_CACHE_NONE; |
3493 | 3490 | ||
3494 | return obj; |
3491 | return obj; |
3495 | } |
3492 | } |
3496 | 3493 | ||
3497 | int i915_gem_init_object(struct drm_gem_object *obj) |
3494 | int i915_gem_init_object(struct drm_gem_object *obj) |
3498 | { |
3495 | { |
3499 | BUG(); |
3496 | BUG(); |
3500 | 3497 | ||
3501 | return 0; |
3498 | return 0; |
3502 | } |
3499 | } |
3503 | 3500 | ||
3504 | void i915_gem_free_object(struct drm_gem_object *gem_obj) |
3501 | void i915_gem_free_object(struct drm_gem_object *gem_obj) |
3505 | { |
3502 | { |
3506 | struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); |
3503 | struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); |
3507 | struct drm_device *dev = obj->base.dev; |
3504 | struct drm_device *dev = obj->base.dev; |
3508 | drm_i915_private_t *dev_priv = dev->dev_private; |
3505 | drm_i915_private_t *dev_priv = dev->dev_private; |
3509 | 3506 | ||
3510 | trace_i915_gem_object_destroy(obj); |
3507 | trace_i915_gem_object_destroy(obj); |
3511 | 3508 | ||
3512 | // if (obj->phys_obj) |
3509 | // if (obj->phys_obj) |
3513 | // i915_gem_detach_phys_object(dev, obj); |
3510 | // i915_gem_detach_phys_object(dev, obj); |
3514 | printf("%s obj %p\n", __FUNCTION__, obj); |
3511 | // printf("%s obj %p\n", __FUNCTION__, obj); |
3515 | 3512 | ||
3516 | obj->pin_count = 0; |
3513 | obj->pin_count = 0; |
3517 | if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { |
3514 | if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { |
3518 | bool was_interruptible; |
3515 | bool was_interruptible; |
3519 | 3516 | ||
3520 | was_interruptible = dev_priv->mm.interruptible; |
3517 | was_interruptible = dev_priv->mm.interruptible; |
3521 | dev_priv->mm.interruptible = false; |
3518 | dev_priv->mm.interruptible = false; |
3522 | 3519 | ||
3523 | WARN_ON(i915_gem_object_unbind(obj)); |
3520 | WARN_ON(i915_gem_object_unbind(obj)); |
3524 | 3521 | ||
3525 | dev_priv->mm.interruptible = was_interruptible; |
3522 | dev_priv->mm.interruptible = was_interruptible; |
3526 | } |
3523 | } |
3527 | 3524 | ||
3528 | obj->pages_pin_count = 0; |
3525 | obj->pages_pin_count = 0; |
3529 | i915_gem_object_put_pages(obj); |
3526 | i915_gem_object_put_pages(obj); |
3530 | // i915_gem_object_free_mmap_offset(obj); |
3527 | // i915_gem_object_free_mmap_offset(obj); |
3531 | 3528 | ||
3532 | BUG_ON(obj->pages); |
3529 | BUG_ON(obj->pages); |
3533 | 3530 | ||
3534 | // if (obj->base.import_attach) |
3531 | // if (obj->base.import_attach) |
3535 | // drm_prime_gem_destroy(&obj->base, NULL); |
3532 | // drm_prime_gem_destroy(&obj->base, NULL); |
3536 | 3533 | ||
3537 | if(obj->base.filp != NULL) |
3534 | if(obj->base.filp != NULL) |
3538 | { |
3535 | { |
3539 | printf("filp %p\n", obj->base.filp); |
3536 | // printf("filp %p\n", obj->base.filp); |
3540 | shmem_file_delete(obj->base.filp); |
3537 | shmem_file_delete(obj->base.filp); |
3541 | } |
3538 | } |
3542 | 3539 | ||
3543 | drm_gem_object_release(&obj->base); |
3540 | drm_gem_object_release(&obj->base); |
3544 | i915_gem_info_remove_obj(dev_priv, obj->base.size); |
3541 | i915_gem_info_remove_obj(dev_priv, obj->base.size); |
3545 | 3542 | ||
3546 | kfree(obj->bit_17); |
3543 | kfree(obj->bit_17); |
3547 | kfree(obj); |
3544 | kfree(obj); |
3548 | } |
3545 | } |
3549 | 3546 | ||
3550 | #if 0 |
3547 | #if 0 |
3551 | int |
3548 | int |
3552 | i915_gem_idle(struct drm_device *dev) |
3549 | i915_gem_idle(struct drm_device *dev) |
3553 | { |
3550 | { |
3554 | drm_i915_private_t *dev_priv = dev->dev_private; |
3551 | drm_i915_private_t *dev_priv = dev->dev_private; |
3555 | int ret; |
3552 | int ret; |
3556 | 3553 | ||
3557 | mutex_lock(&dev->struct_mutex); |
3554 | mutex_lock(&dev->struct_mutex); |
3558 | 3555 | ||
3559 | if (dev_priv->mm.suspended) { |
3556 | if (dev_priv->mm.suspended) { |
3560 | mutex_unlock(&dev->struct_mutex); |
3557 | mutex_unlock(&dev->struct_mutex); |
3561 | return 0; |
3558 | return 0; |
3562 | } |
3559 | } |
3563 | 3560 | ||
3564 | ret = i915_gpu_idle(dev); |
3561 | ret = i915_gpu_idle(dev); |
3565 | if (ret) { |
3562 | if (ret) { |
3566 | mutex_unlock(&dev->struct_mutex); |
3563 | mutex_unlock(&dev->struct_mutex); |
3567 | return ret; |
3564 | return ret; |
3568 | } |
3565 | } |
3569 | i915_gem_retire_requests(dev); |
3566 | i915_gem_retire_requests(dev); |
3570 | 3567 | ||
3571 | i915_gem_reset_fences(dev); |
3568 | i915_gem_reset_fences(dev); |
3572 | 3569 | ||
3573 | /* Hack! Don't let anybody do execbuf while we don't control the chip. |
3570 | /* Hack! Don't let anybody do execbuf while we don't control the chip. |
3574 | * We need to replace this with a semaphore, or something. |
3571 | * We need to replace this with a semaphore, or something. |
3575 | * And not confound mm.suspended! |
3572 | * And not confound mm.suspended! |
3576 | */ |
3573 | */ |
3577 | dev_priv->mm.suspended = 1; |
3574 | dev_priv->mm.suspended = 1; |
3578 | del_timer_sync(&dev_priv->hangcheck_timer); |
3575 | del_timer_sync(&dev_priv->hangcheck_timer); |
3579 | 3576 | ||
3580 | i915_kernel_lost_context(dev); |
3577 | i915_kernel_lost_context(dev); |
3581 | i915_gem_cleanup_ringbuffer(dev); |
3578 | i915_gem_cleanup_ringbuffer(dev); |
3582 | 3579 | ||
3583 | mutex_unlock(&dev->struct_mutex); |
3580 | mutex_unlock(&dev->struct_mutex); |
3584 | 3581 | ||
3585 | /* Cancel the retire work handler, which should be idle now. */ |
3582 | /* Cancel the retire work handler, which should be idle now. */ |
3586 | cancel_delayed_work_sync(&dev_priv->mm.retire_work); |
3583 | cancel_delayed_work_sync(&dev_priv->mm.retire_work); |
3587 | 3584 | ||
3588 | return 0; |
3585 | return 0; |
3589 | } |
3586 | } |
3590 | #endif |
3587 | #endif |
3591 | 3588 | ||
3592 | void i915_gem_l3_remap(struct drm_device *dev) |
3589 | void i915_gem_l3_remap(struct drm_device *dev) |
3593 | { |
3590 | { |
3594 | drm_i915_private_t *dev_priv = dev->dev_private; |
3591 | drm_i915_private_t *dev_priv = dev->dev_private; |
3595 | u32 misccpctl; |
3592 | u32 misccpctl; |
3596 | int i; |
3593 | int i; |
3597 | 3594 | ||
3598 | if (!IS_IVYBRIDGE(dev)) |
3595 | if (!IS_IVYBRIDGE(dev)) |
3599 | return; |
3596 | return; |
3600 | 3597 | ||
3601 | if (!dev_priv->l3_parity.remap_info) |
3598 | if (!dev_priv->l3_parity.remap_info) |
3602 | return; |
3599 | return; |
3603 | 3600 | ||
3604 | misccpctl = I915_READ(GEN7_MISCCPCTL); |
3601 | misccpctl = I915_READ(GEN7_MISCCPCTL); |
3605 | I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); |
3602 | I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); |
3606 | POSTING_READ(GEN7_MISCCPCTL); |
3603 | POSTING_READ(GEN7_MISCCPCTL); |
3607 | 3604 | ||
3608 | for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { |
3605 | for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { |
3609 | u32 remap = I915_READ(GEN7_L3LOG_BASE + i); |
3606 | u32 remap = I915_READ(GEN7_L3LOG_BASE + i); |
3610 | if (remap && remap != dev_priv->l3_parity.remap_info[i/4]) |
3607 | if (remap && remap != dev_priv->l3_parity.remap_info[i/4]) |
3611 | DRM_DEBUG("0x%x was already programmed to %x\n", |
3608 | DRM_DEBUG("0x%x was already programmed to %x\n", |
3612 | GEN7_L3LOG_BASE + i, remap); |
3609 | GEN7_L3LOG_BASE + i, remap); |
3613 | if (remap && !dev_priv->l3_parity.remap_info[i/4]) |
3610 | if (remap && !dev_priv->l3_parity.remap_info[i/4]) |
3614 | DRM_DEBUG_DRIVER("Clearing remapped register\n"); |
3611 | DRM_DEBUG_DRIVER("Clearing remapped register\n"); |
3615 | I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]); |
3612 | I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]); |
3616 | } |
3613 | } |
3617 | 3614 | ||
3618 | /* Make sure all the writes land before disabling dop clock gating */ |
3615 | /* Make sure all the writes land before disabling dop clock gating */ |
3619 | POSTING_READ(GEN7_L3LOG_BASE); |
3616 | POSTING_READ(GEN7_L3LOG_BASE); |
3620 | 3617 | ||
3621 | I915_WRITE(GEN7_MISCCPCTL, misccpctl); |
3618 | I915_WRITE(GEN7_MISCCPCTL, misccpctl); |
3622 | } |
3619 | } |
3623 | 3620 | ||
3624 | void i915_gem_init_swizzling(struct drm_device *dev) |
3621 | void i915_gem_init_swizzling(struct drm_device *dev) |
3625 | { |
3622 | { |
3626 | drm_i915_private_t *dev_priv = dev->dev_private; |
3623 | drm_i915_private_t *dev_priv = dev->dev_private; |
3627 | 3624 | ||
3628 | if (INTEL_INFO(dev)->gen < 5 || |
3625 | if (INTEL_INFO(dev)->gen < 5 || |
3629 | dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) |
3626 | dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) |
3630 | return; |
3627 | return; |
3631 | 3628 | ||
3632 | I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | |
3629 | I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | |
3633 | DISP_TILE_SURFACE_SWIZZLING); |
3630 | DISP_TILE_SURFACE_SWIZZLING); |
3634 | 3631 | ||
3635 | if (IS_GEN5(dev)) |
3632 | if (IS_GEN5(dev)) |
3636 | return; |
3633 | return; |
3637 | 3634 | ||
3638 | I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); |
3635 | I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); |
3639 | if (IS_GEN6(dev)) |
3636 | if (IS_GEN6(dev)) |
3640 | I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); |
3637 | I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); |
3641 | else |
3638 | else |
3642 | I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); |
3639 | I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); |
3643 | } |
3640 | } |
3644 | 3641 | ||
3645 | static bool |
3642 | static bool |
3646 | intel_enable_blt(struct drm_device *dev) |
3643 | intel_enable_blt(struct drm_device *dev) |
3647 | { |
3644 | { |
3648 | if (!HAS_BLT(dev)) |
3645 | if (!HAS_BLT(dev)) |
3649 | return false; |
3646 | return false; |
3650 | 3647 | ||
3651 | /* The blitter was dysfunctional on early prototypes */ |
3648 | /* The blitter was dysfunctional on early prototypes */ |
3652 | if (IS_GEN6(dev) && dev->pdev->revision < 8) { |
3649 | if (IS_GEN6(dev) && dev->pdev->revision < 8) { |
3653 | DRM_INFO("BLT not supported on this pre-production hardware;" |
3650 | DRM_INFO("BLT not supported on this pre-production hardware;" |
3654 | " graphics performance will be degraded.\n"); |
3651 | " graphics performance will be degraded.\n"); |
3655 | return false; |
3652 | return false; |
3656 | } |
3653 | } |
3657 | 3654 | ||
3658 | return true; |
3655 | return true; |
3659 | } |
3656 | } |
3660 | 3657 | ||
3661 | int |
3658 | int |
3662 | i915_gem_init_hw(struct drm_device *dev) |
3659 | i915_gem_init_hw(struct drm_device *dev) |
3663 | { |
3660 | { |
3664 | drm_i915_private_t *dev_priv = dev->dev_private; |
3661 | drm_i915_private_t *dev_priv = dev->dev_private; |
3665 | int ret; |
3662 | int ret; |
3666 | 3663 | ||
3667 | if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) |
3664 | if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) |
3668 | return -EIO; |
3665 | return -EIO; |
3669 | 3666 | ||
3670 | if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) |
3667 | if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) |
3671 | I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); |
3668 | I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); |
3672 | 3669 | ||
3673 | i915_gem_l3_remap(dev); |
3670 | i915_gem_l3_remap(dev); |
3674 | 3671 | ||
3675 | i915_gem_init_swizzling(dev); |
3672 | i915_gem_init_swizzling(dev); |
3676 | 3673 | ||
3677 | ret = intel_init_render_ring_buffer(dev); |
3674 | ret = intel_init_render_ring_buffer(dev); |
3678 | if (ret) |
3675 | if (ret) |
3679 | return ret; |
3676 | return ret; |
3680 | 3677 | ||
3681 | if (HAS_BSD(dev)) { |
3678 | if (HAS_BSD(dev)) { |
3682 | ret = intel_init_bsd_ring_buffer(dev); |
3679 | ret = intel_init_bsd_ring_buffer(dev); |
3683 | if (ret) |
3680 | if (ret) |
3684 | goto cleanup_render_ring; |
3681 | goto cleanup_render_ring; |
3685 | } |
3682 | } |
3686 | 3683 | ||
3687 | if (intel_enable_blt(dev)) { |
3684 | if (intel_enable_blt(dev)) { |
3688 | ret = intel_init_blt_ring_buffer(dev); |
3685 | ret = intel_init_blt_ring_buffer(dev); |
3689 | if (ret) |
3686 | if (ret) |
3690 | goto cleanup_bsd_ring; |
3687 | goto cleanup_bsd_ring; |
3691 | } |
3688 | } |
3692 | 3689 | ||
3693 | dev_priv->next_seqno = 1; |
3690 | dev_priv->next_seqno = 1; |
3694 | 3691 | ||
3695 | /* |
3692 | /* |
3696 | * XXX: There was some w/a described somewhere suggesting loading |
3693 | * XXX: There was some w/a described somewhere suggesting loading |
3697 | * contexts before PPGTT. |
3694 | * contexts before PPGTT. |
3698 | */ |
3695 | */ |
3699 | i915_gem_context_init(dev); |
3696 | i915_gem_context_init(dev); |
3700 | i915_gem_init_ppgtt(dev); |
3697 | i915_gem_init_ppgtt(dev); |
3701 | 3698 | ||
3702 | return 0; |
3699 | return 0; |
3703 | 3700 | ||
3704 | cleanup_bsd_ring: |
3701 | cleanup_bsd_ring: |
3705 | intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); |
3702 | intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); |
3706 | cleanup_render_ring: |
3703 | cleanup_render_ring: |
3707 | intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); |
3704 | intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); |
3708 | return ret; |
3705 | return ret; |
3709 | } |
3706 | } |
3710 | 3707 | ||
3711 | static bool |
3708 | static bool |
3712 | intel_enable_ppgtt(struct drm_device *dev) |
3709 | intel_enable_ppgtt(struct drm_device *dev) |
3713 | { |
3710 | { |
3714 | if (i915_enable_ppgtt >= 0) |
3711 | if (i915_enable_ppgtt >= 0) |
3715 | return i915_enable_ppgtt; |
3712 | return i915_enable_ppgtt; |
3716 | 3713 | ||
3717 | #ifdef CONFIG_INTEL_IOMMU |
3714 | #ifdef CONFIG_INTEL_IOMMU |
3718 | /* Disable ppgtt on SNB if VT-d is on. */ |
3715 | /* Disable ppgtt on SNB if VT-d is on. */ |
3719 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
3716 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
3720 | return false; |
3717 | return false; |
3721 | #endif |
3718 | #endif |
3722 | 3719 | ||
3723 | return true; |
3720 | return true; |
3724 | } |
3721 | } |
3725 | 3722 | ||
3726 | #define LFB_SIZE 0xC00000 |
3723 | #define LFB_SIZE 0xC00000 |
3727 | 3724 | ||
3728 | int i915_gem_init(struct drm_device *dev) |
3725 | int i915_gem_init(struct drm_device *dev) |
3729 | { |
3726 | { |
3730 | struct drm_i915_private *dev_priv = dev->dev_private; |
3727 | struct drm_i915_private *dev_priv = dev->dev_private; |
3731 | unsigned long gtt_size, mappable_size; |
3728 | unsigned long gtt_size, mappable_size; |
3732 | int ret; |
3729 | int ret; |
3733 | 3730 | ||
3734 | gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; |
3731 | gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; |
3735 | mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; |
3732 | mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; |
3736 | 3733 | ||
3737 | mutex_lock(&dev->struct_mutex); |
3734 | mutex_lock(&dev->struct_mutex); |
3738 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
3735 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
3739 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
3736 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
3740 | * aperture accordingly when using aliasing ppgtt. */ |
3737 | * aperture accordingly when using aliasing ppgtt. */ |
3741 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
3738 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
3742 | 3739 | ||
3743 | i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE); |
3740 | i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE); |
3744 | 3741 | ||
3745 | ret = i915_gem_init_aliasing_ppgtt(dev); |
3742 | ret = i915_gem_init_aliasing_ppgtt(dev); |
3746 | if (ret) { |
3743 | if (ret) { |
3747 | mutex_unlock(&dev->struct_mutex); |
3744 | mutex_unlock(&dev->struct_mutex); |
3748 | return ret; |
3745 | return ret; |
3749 | } |
3746 | } |
3750 | } else { |
3747 | } else { |
3751 | /* Let GEM Manage all of the aperture. |
3748 | /* Let GEM Manage all of the aperture. |
3752 | * |
3749 | * |
3753 | * However, leave one page at the end still bound to the scratch |
3750 | * However, leave one page at the end still bound to the scratch |
3754 | * page. There are a number of places where the hardware |
3751 | * page. There are a number of places where the hardware |
3755 | * apparently prefetches past the end of the object, and we've |
3752 | * apparently prefetches past the end of the object, and we've |
3756 | * seen multiple hangs with the GPU head pointer stuck in a |
3753 | * seen multiple hangs with the GPU head pointer stuck in a |
3757 | * batchbuffer bound at the last page of the aperture. One page |
3754 | * batchbuffer bound at the last page of the aperture. One page |
3758 | * should be enough to keep any prefetching inside of the |
3755 | * should be enough to keep any prefetching inside of the |
3759 | * aperture. |
3756 | * aperture. |
3760 | */ |
3757 | */ |
3761 | i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE); |
3758 | i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE); |
3762 | } |
3759 | } |
3763 | 3760 | ||
3764 | ret = i915_gem_init_hw(dev); |
3761 | ret = i915_gem_init_hw(dev); |
3765 | mutex_unlock(&dev->struct_mutex); |
3762 | mutex_unlock(&dev->struct_mutex); |
3766 | if (ret) { |
3763 | if (ret) { |
3767 | i915_gem_cleanup_aliasing_ppgtt(dev); |
3764 | i915_gem_cleanup_aliasing_ppgtt(dev); |
3768 | return ret; |
3765 | return ret; |
3769 | } |
3766 | } |
3770 | 3767 | ||
3771 | return 0; |
3768 | return 0; |
3772 | } |
3769 | } |
3773 | 3770 | ||
3774 | void |
3771 | void |
3775 | i915_gem_cleanup_ringbuffer(struct drm_device *dev) |
3772 | i915_gem_cleanup_ringbuffer(struct drm_device *dev) |
3776 | { |
3773 | { |
3777 | drm_i915_private_t *dev_priv = dev->dev_private; |
3774 | drm_i915_private_t *dev_priv = dev->dev_private; |
3778 | struct intel_ring_buffer *ring; |
3775 | struct intel_ring_buffer *ring; |
3779 | int i; |
3776 | int i; |
3780 | 3777 | ||
3781 | for_each_ring(ring, dev_priv, i) |
3778 | for_each_ring(ring, dev_priv, i) |
3782 | intel_cleanup_ring_buffer(ring); |
3779 | intel_cleanup_ring_buffer(ring); |
3783 | } |
3780 | } |
3784 | 3781 | ||
3785 | #if 0 |
3782 | #if 0 |
3786 | 3783 | ||
3787 | int |
3784 | int |
3788 | i915_gem_entervt_ioctl(struct drm_device *dev, void *data, |
3785 | i915_gem_entervt_ioctl(struct drm_device *dev, void *data, |
3789 | struct drm_file *file_priv) |
3786 | struct drm_file *file_priv) |
3790 | { |
3787 | { |
3791 | drm_i915_private_t *dev_priv = dev->dev_private; |
3788 | drm_i915_private_t *dev_priv = dev->dev_private; |
3792 | int ret; |
3789 | int ret; |
3793 | 3790 | ||
3794 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
3791 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
3795 | return 0; |
3792 | return 0; |
3796 | 3793 | ||
3797 | if (atomic_read(&dev_priv->mm.wedged)) { |
3794 | if (atomic_read(&dev_priv->mm.wedged)) { |
3798 | DRM_ERROR("Reenabling wedged hardware, good luck\n"); |
3795 | DRM_ERROR("Reenabling wedged hardware, good luck\n"); |
3799 | atomic_set(&dev_priv->mm.wedged, 0); |
3796 | atomic_set(&dev_priv->mm.wedged, 0); |
3800 | } |
3797 | } |
3801 | 3798 | ||
3802 | mutex_lock(&dev->struct_mutex); |
3799 | mutex_lock(&dev->struct_mutex); |
3803 | dev_priv->mm.suspended = 0; |
3800 | dev_priv->mm.suspended = 0; |
3804 | 3801 | ||
3805 | ret = i915_gem_init_hw(dev); |
3802 | ret = i915_gem_init_hw(dev); |
3806 | if (ret != 0) { |
3803 | if (ret != 0) { |
3807 | mutex_unlock(&dev->struct_mutex); |
3804 | mutex_unlock(&dev->struct_mutex); |
3808 | return ret; |
3805 | return ret; |
3809 | } |
3806 | } |
3810 | 3807 | ||
3811 | BUG_ON(!list_empty(&dev_priv->mm.active_list)); |
3808 | BUG_ON(!list_empty(&dev_priv->mm.active_list)); |
3812 | mutex_unlock(&dev->struct_mutex); |
3809 | mutex_unlock(&dev->struct_mutex); |
3813 | 3810 | ||
3814 | ret = drm_irq_install(dev); |
3811 | ret = drm_irq_install(dev); |
3815 | if (ret) |
3812 | if (ret) |
3816 | goto cleanup_ringbuffer; |
3813 | goto cleanup_ringbuffer; |
3817 | 3814 | ||
3818 | return 0; |
3815 | return 0; |
3819 | 3816 | ||
3820 | cleanup_ringbuffer: |
3817 | cleanup_ringbuffer: |
3821 | mutex_lock(&dev->struct_mutex); |
3818 | mutex_lock(&dev->struct_mutex); |
3822 | i915_gem_cleanup_ringbuffer(dev); |
3819 | i915_gem_cleanup_ringbuffer(dev); |
3823 | dev_priv->mm.suspended = 1; |
3820 | dev_priv->mm.suspended = 1; |
3824 | mutex_unlock(&dev->struct_mutex); |
3821 | mutex_unlock(&dev->struct_mutex); |
3825 | 3822 | ||
3826 | return ret; |
3823 | return ret; |
3827 | } |
3824 | } |
3828 | 3825 | ||
3829 | int |
3826 | int |
3830 | i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, |
3827 | i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, |
3831 | struct drm_file *file_priv) |
3828 | struct drm_file *file_priv) |
3832 | { |
3829 | { |
3833 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
3830 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
3834 | return 0; |
3831 | return 0; |
3835 | 3832 | ||
3836 | drm_irq_uninstall(dev); |
3833 | drm_irq_uninstall(dev); |
3837 | return i915_gem_idle(dev); |
3834 | return i915_gem_idle(dev); |
3838 | } |
3835 | } |
3839 | 3836 | ||
3840 | void |
3837 | void |
3841 | i915_gem_lastclose(struct drm_device *dev) |
3838 | i915_gem_lastclose(struct drm_device *dev) |
3842 | { |
3839 | { |
3843 | int ret; |
3840 | int ret; |
3844 | 3841 | ||
3845 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
3842 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
3846 | return; |
3843 | return; |
3847 | 3844 | ||
3848 | ret = i915_gem_idle(dev); |
3845 | ret = i915_gem_idle(dev); |
3849 | if (ret) |
3846 | if (ret) |
3850 | DRM_ERROR("failed to idle hardware: %d\n", ret); |
3847 | DRM_ERROR("failed to idle hardware: %d\n", ret); |
3851 | } |
3848 | } |
3852 | #endif |
3849 | #endif |
3853 | 3850 | ||
3854 | static void |
3851 | static void |
3855 | init_ring_lists(struct intel_ring_buffer *ring) |
3852 | init_ring_lists(struct intel_ring_buffer *ring) |
3856 | { |
3853 | { |
3857 | INIT_LIST_HEAD(&ring->active_list); |
3854 | INIT_LIST_HEAD(&ring->active_list); |
3858 | INIT_LIST_HEAD(&ring->request_list); |
3855 | INIT_LIST_HEAD(&ring->request_list); |
3859 | } |
3856 | } |
3860 | 3857 | ||
3861 | void |
3858 | void |
3862 | i915_gem_load(struct drm_device *dev) |
3859 | i915_gem_load(struct drm_device *dev) |
3863 | { |
3860 | { |
3864 | int i; |
3861 | int i; |
3865 | drm_i915_private_t *dev_priv = dev->dev_private; |
3862 | drm_i915_private_t *dev_priv = dev->dev_private; |
3866 | 3863 | ||
3867 | INIT_LIST_HEAD(&dev_priv->mm.active_list); |
3864 | INIT_LIST_HEAD(&dev_priv->mm.active_list); |
3868 | INIT_LIST_HEAD(&dev_priv->mm.inactive_list); |
3865 | INIT_LIST_HEAD(&dev_priv->mm.inactive_list); |
3869 | INIT_LIST_HEAD(&dev_priv->mm.unbound_list); |
3866 | INIT_LIST_HEAD(&dev_priv->mm.unbound_list); |
3870 | INIT_LIST_HEAD(&dev_priv->mm.bound_list); |
3867 | INIT_LIST_HEAD(&dev_priv->mm.bound_list); |
3871 | INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
3868 | INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
3872 | for (i = 0; i < I915_NUM_RINGS; i++) |
3869 | for (i = 0; i < I915_NUM_RINGS; i++) |
3873 | init_ring_lists(&dev_priv->ring[i]); |
3870 | init_ring_lists(&dev_priv->ring[i]); |
3874 | for (i = 0; i < I915_MAX_NUM_FENCES; i++) |
3871 | for (i = 0; i < I915_MAX_NUM_FENCES; i++) |
3875 | INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); |
3872 | INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); |
3876 | INIT_DELAYED_WORK(&dev_priv->mm.retire_work, |
3873 | INIT_DELAYED_WORK(&dev_priv->mm.retire_work, |
3877 | i915_gem_retire_work_handler); |
3874 | i915_gem_retire_work_handler); |
3878 | 3875 | ||
3879 | /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ |
3876 | /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ |
3880 | if (IS_GEN3(dev)) { |
3877 | if (IS_GEN3(dev)) { |
3881 | I915_WRITE(MI_ARB_STATE, |
3878 | I915_WRITE(MI_ARB_STATE, |
3882 | _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); |
3879 | _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); |
3883 | } |
3880 | } |
3884 | 3881 | ||
3885 | dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
3882 | dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
3886 | 3883 | ||
3887 | if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
3884 | if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
3888 | dev_priv->num_fence_regs = 16; |
3885 | dev_priv->num_fence_regs = 16; |
3889 | else |
3886 | else |
3890 | dev_priv->num_fence_regs = 8; |
3887 | dev_priv->num_fence_regs = 8; |
3891 | 3888 | ||
3892 | /* Initialize fence registers to zero */ |
3889 | /* Initialize fence registers to zero */ |
3893 | i915_gem_reset_fences(dev); |
3890 | i915_gem_reset_fences(dev); |
3894 | 3891 | ||
3895 | i915_gem_detect_bit_6_swizzle(dev); |
3892 | i915_gem_detect_bit_6_swizzle(dev); |
3896 | 3893 | ||
3897 | dev_priv->mm.interruptible = true; |
3894 | dev_priv->mm.interruptible = true; |
3898 | 3895 | ||
3899 | // dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink; |
3896 | // dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink; |
3900 | // dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS; |
3897 | // dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS; |
3901 | // register_shrinker(&dev_priv->mm.inactive_shrinker); |
3898 | // register_shrinker(&dev_priv->mm.inactive_shrinker); |
3902 | }>>><>><>>>>>><>>>>=>>><>><>>><>><>><>><>><>><> |
3899 | }>>><>><>>>>>><>>>>=>>><>><>>><>><>><>><>><>><> |
3903 | >< |
3900 | >< |
3904 | >><>=>>>>=><=>> |
3901 | >><>=>>>>=><=>> |
3905 | //> |
3902 | //> |
3906 | //>>><>=>>><>=>>>6)><6)>6) |
3903 | //>>><>=>>><>=>>>6)><6)>6) |
3907 | #define><6) |
3904 | #define><6) |
3908 | #define>6)><6)>6) |
3905 | #define>6)><6)>6) |
3909 | #define><6) |
3906 | #define><6) |
3910 | #define> |
3907 | #define> |