Rev 3746 | Rev 4104 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 3746 | Rev 3764 | ||
---|---|---|---|
1 | /* |
1 | /* |
2 | * Copyright © 2010 Daniel Vetter |
2 | * Copyright © 2010 Daniel Vetter |
3 | * |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
13 | * Software. |
14 | * |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
21 | * IN THE SOFTWARE. |
22 | * |
22 | * |
23 | */ |
23 | */ |
24 | 24 | ||
25 | 25 | ||
26 | #define AGP_NORMAL_MEMORY 0 |
26 | #define AGP_NORMAL_MEMORY 0 |
27 | 27 | ||
28 | #define AGP_USER_TYPES (1 << 16) |
28 | #define AGP_USER_TYPES (1 << 16) |
29 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
29 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
30 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
30 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
31 | 31 | ||
32 | #include |
32 | #include |
33 | #include |
33 | #include |
34 | #include "i915_drv.h" |
34 | #include "i915_drv.h" |
35 | #include "i915_trace.h" |
35 | #include "i915_trace.h" |
36 | #include "intel_drv.h" |
36 | #include "intel_drv.h" |
37 | 37 | ||
38 | typedef uint32_t gen6_gtt_pte_t; |
38 | typedef uint32_t gen6_gtt_pte_t; |
39 | 39 | ||
40 | /* PPGTT stuff */ |
40 | /* PPGTT stuff */ |
41 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
41 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
42 | 42 | ||
43 | #define GEN6_PDE_VALID (1 << 0) |
43 | #define GEN6_PDE_VALID (1 << 0) |
44 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
44 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
45 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
45 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
46 | 46 | ||
47 | #define GEN6_PTE_VALID (1 << 0) |
47 | #define GEN6_PTE_VALID (1 << 0) |
48 | #define GEN6_PTE_UNCACHED (1 << 1) |
48 | #define GEN6_PTE_UNCACHED (1 << 1) |
49 | #define HSW_PTE_UNCACHED (0) |
49 | #define HSW_PTE_UNCACHED (0) |
50 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
50 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
51 | #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) |
51 | #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) |
52 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
52 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
53 | 53 | ||
54 | static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev, |
54 | static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev, |
55 | dma_addr_t addr, |
55 | dma_addr_t addr, |
56 | enum i915_cache_level level) |
56 | enum i915_cache_level level) |
57 | { |
57 | { |
58 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
58 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
59 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
59 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
60 | 60 | ||
61 | switch (level) { |
61 | switch (level) { |
62 | case I915_CACHE_LLC_MLC: |
62 | case I915_CACHE_LLC_MLC: |
63 | /* Haswell doesn't set L3 this way */ |
63 | /* Haswell doesn't set L3 this way */ |
64 | if (IS_HASWELL(dev)) |
64 | if (IS_HASWELL(dev)) |
65 | pte |= GEN6_PTE_CACHE_LLC; |
65 | pte |= GEN6_PTE_CACHE_LLC; |
66 | else |
66 | else |
67 | pte |= GEN6_PTE_CACHE_LLC_MLC; |
67 | pte |= GEN6_PTE_CACHE_LLC_MLC; |
68 | break; |
68 | break; |
69 | case I915_CACHE_LLC: |
69 | case I915_CACHE_LLC: |
70 | pte |= GEN6_PTE_CACHE_LLC; |
70 | pte |= GEN6_PTE_CACHE_LLC; |
71 | break; |
71 | break; |
72 | case I915_CACHE_NONE: |
72 | case I915_CACHE_NONE: |
73 | if (IS_HASWELL(dev)) |
73 | if (IS_HASWELL(dev)) |
74 | pte |= HSW_PTE_UNCACHED; |
74 | pte |= HSW_PTE_UNCACHED; |
75 | else |
75 | else |
76 | pte |= GEN6_PTE_UNCACHED; |
76 | pte |= GEN6_PTE_UNCACHED; |
77 | break; |
77 | break; |
78 | default: |
78 | default: |
79 | BUG(); |
79 | BUG(); |
80 | } |
80 | } |
81 | 81 | ||
82 | return pte; |
82 | return pte; |
83 | } |
83 | } |
84 | 84 | ||
85 | static int gen6_ppgtt_enable(struct drm_device *dev) |
85 | static int gen6_ppgtt_enable(struct drm_device *dev) |
86 | { |
86 | { |
87 | drm_i915_private_t *dev_priv = dev->dev_private; |
87 | drm_i915_private_t *dev_priv = dev->dev_private; |
88 | uint32_t pd_offset; |
88 | uint32_t pd_offset; |
89 | struct intel_ring_buffer *ring; |
89 | struct intel_ring_buffer *ring; |
90 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
90 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
91 | gen6_gtt_pte_t __iomem *pd_addr; |
91 | gen6_gtt_pte_t __iomem *pd_addr; |
92 | uint32_t pd_entry; |
92 | uint32_t pd_entry; |
93 | int i; |
93 | int i; |
94 | 94 | ||
95 | pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + |
95 | pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + |
96 | ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); |
96 | ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); |
97 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
97 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
98 | dma_addr_t pt_addr; |
98 | dma_addr_t pt_addr; |
99 | 99 | ||
100 | pt_addr = ppgtt->pt_dma_addr[i]; |
100 | pt_addr = ppgtt->pt_dma_addr[i]; |
101 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
101 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
102 | pd_entry |= GEN6_PDE_VALID; |
102 | pd_entry |= GEN6_PDE_VALID; |
103 | 103 | ||
104 | writel(pd_entry, pd_addr + i); |
104 | writel(pd_entry, pd_addr + i); |
105 | } |
105 | } |
106 | readl(pd_addr); |
106 | readl(pd_addr); |
107 | 107 | ||
108 | pd_offset = ppgtt->pd_offset; |
108 | pd_offset = ppgtt->pd_offset; |
109 | pd_offset /= 64; /* in cachelines, */ |
109 | pd_offset /= 64; /* in cachelines, */ |
110 | pd_offset <<= 16; |
110 | pd_offset <<= 16; |
111 | 111 | ||
112 | if (INTEL_INFO(dev)->gen == 6) { |
112 | if (INTEL_INFO(dev)->gen == 6) { |
113 | uint32_t ecochk, gab_ctl, ecobits; |
113 | uint32_t ecochk, gab_ctl, ecobits; |
114 | 114 | ||
115 | ecobits = I915_READ(GAC_ECO_BITS); |
115 | ecobits = I915_READ(GAC_ECO_BITS); |
116 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | |
116 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | |
117 | ECOBITS_PPGTT_CACHE64B); |
117 | ECOBITS_PPGTT_CACHE64B); |
118 | 118 | ||
119 | gab_ctl = I915_READ(GAB_CTL); |
119 | gab_ctl = I915_READ(GAB_CTL); |
120 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
120 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
121 | 121 | ||
122 | ecochk = I915_READ(GAM_ECOCHK); |
122 | ecochk = I915_READ(GAM_ECOCHK); |
123 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
123 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
124 | ECOCHK_PPGTT_CACHE64B); |
124 | ECOCHK_PPGTT_CACHE64B); |
125 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
125 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
126 | } else if (INTEL_INFO(dev)->gen >= 7) { |
126 | } else if (INTEL_INFO(dev)->gen >= 7) { |
127 | uint32_t ecochk, ecobits; |
127 | uint32_t ecochk, ecobits; |
128 | 128 | ||
129 | ecobits = I915_READ(GAC_ECO_BITS); |
129 | ecobits = I915_READ(GAC_ECO_BITS); |
130 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
130 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
131 | 131 | ||
132 | ecochk = I915_READ(GAM_ECOCHK); |
132 | ecochk = I915_READ(GAM_ECOCHK); |
133 | if (IS_HASWELL(dev)) { |
133 | if (IS_HASWELL(dev)) { |
134 | ecochk |= ECOCHK_PPGTT_WB_HSW; |
134 | ecochk |= ECOCHK_PPGTT_WB_HSW; |
135 | } else { |
135 | } else { |
136 | ecochk |= ECOCHK_PPGTT_LLC_IVB; |
136 | ecochk |= ECOCHK_PPGTT_LLC_IVB; |
137 | ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; |
137 | ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; |
138 | } |
138 | } |
139 | I915_WRITE(GAM_ECOCHK, ecochk); |
139 | I915_WRITE(GAM_ECOCHK, ecochk); |
140 | /* GFX_MODE is per-ring on gen7+ */ |
140 | /* GFX_MODE is per-ring on gen7+ */ |
141 | } |
141 | } |
142 | 142 | ||
143 | for_each_ring(ring, dev_priv, i) { |
143 | for_each_ring(ring, dev_priv, i) { |
144 | if (INTEL_INFO(dev)->gen >= 7) |
144 | if (INTEL_INFO(dev)->gen >= 7) |
145 | I915_WRITE(RING_MODE_GEN7(ring), |
145 | I915_WRITE(RING_MODE_GEN7(ring), |
146 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
146 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
147 | 147 | ||
148 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
148 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
149 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
149 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
150 | } |
150 | } |
151 | return 0; |
151 | return 0; |
152 | } |
152 | } |
153 | 153 | ||
154 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
154 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
155 | static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, |
155 | static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, |
156 | unsigned first_entry, |
156 | unsigned first_entry, |
157 | unsigned num_entries) |
157 | unsigned num_entries) |
158 | { |
158 | { |
159 | gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
159 | gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
160 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
160 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
161 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
161 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
162 | unsigned last_pte, i; |
162 | unsigned last_pte, i; |
163 | 163 | ||
164 | scratch_pte = gen6_pte_encode(ppgtt->dev, |
164 | scratch_pte = gen6_pte_encode(ppgtt->dev, |
165 | ppgtt->scratch_page_dma_addr, |
165 | ppgtt->scratch_page_dma_addr, |
166 | I915_CACHE_LLC); |
166 | I915_CACHE_LLC); |
167 | 167 | ||
168 | pt_vaddr = AllocKernelSpace(4096); |
168 | pt_vaddr = AllocKernelSpace(4096); |
169 | 169 | ||
170 | if(pt_vaddr == NULL) |
170 | if(pt_vaddr == NULL) |
171 | return; |
171 | return; |
172 | 172 | ||
173 | while (num_entries) { |
173 | while (num_entries) { |
174 | last_pte = first_pte + num_entries; |
174 | last_pte = first_pte + num_entries; |
175 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
175 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
176 | last_pte = I915_PPGTT_PT_ENTRIES; |
176 | last_pte = I915_PPGTT_PT_ENTRIES; |
177 | 177 | ||
178 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
178 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
179 | 179 | ||
180 | for (i = first_pte; i < last_pte; i++) |
180 | for (i = first_pte; i < last_pte; i++) |
181 | pt_vaddr[i] = scratch_pte; |
181 | pt_vaddr[i] = scratch_pte; |
182 | 182 | ||
183 | num_entries -= last_pte - first_pte; |
183 | num_entries -= last_pte - first_pte; |
184 | first_pte = 0; |
184 | first_pte = 0; |
185 | act_pt++; |
185 | act_pt++; |
186 | }; |
186 | }; |
187 | 187 | ||
188 | FreeKernelSpace(pt_vaddr); |
188 | FreeKernelSpace(pt_vaddr); |
189 | } |
189 | } |
190 | 190 | ||
191 | static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt, |
191 | static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt, |
192 | struct sg_table *pages, |
192 | struct sg_table *pages, |
193 | unsigned first_entry, |
193 | unsigned first_entry, |
194 | enum i915_cache_level cache_level) |
194 | enum i915_cache_level cache_level) |
195 | { |
195 | { |
196 | gen6_gtt_pte_t *pt_vaddr; |
196 | gen6_gtt_pte_t *pt_vaddr; |
197 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
197 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
198 | unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
198 | unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
199 | struct sg_page_iter sg_iter; |
199 | struct sg_page_iter sg_iter; |
200 | dma_addr_t page_addr; |
200 | dma_addr_t page_addr; |
201 | 201 | ||
202 | 202 | ||
203 | pt_vaddr = AllocKernelSpace(4096); |
203 | pt_vaddr = AllocKernelSpace(4096); |
204 | 204 | ||
205 | if(pt_vaddr == NULL) |
205 | if(pt_vaddr == NULL) |
206 | return; |
206 | return; |
207 | 207 | ||
208 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
208 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
209 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
209 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
210 | dma_addr_t page_addr; |
210 | dma_addr_t page_addr; |
211 | 211 | ||
212 | page_addr = sg_page_iter_dma_address(&sg_iter); |
212 | page_addr = sg_page_iter_dma_address(&sg_iter); |
213 | pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr, |
213 | pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr, |
214 | cache_level); |
214 | cache_level); |
215 | if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
215 | if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
216 | act_pt++; |
216 | act_pt++; |
217 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
217 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
218 | act_pte = 0; |
218 | act_pte = 0; |
219 | 219 | ||
220 | } |
220 | } |
221 | } |
221 | } |
222 | FreeKernelSpace(pt_vaddr); |
222 | FreeKernelSpace(pt_vaddr); |
223 | } |
223 | } |
224 | 224 | ||
225 | static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) |
225 | static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) |
226 | { |
226 | { |
227 | int i; |
227 | int i; |
228 | 228 | ||
229 | if (ppgtt->pt_dma_addr) { |
229 | if (ppgtt->pt_dma_addr) { |
230 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
230 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
231 | pci_unmap_page(ppgtt->dev->pdev, |
231 | pci_unmap_page(ppgtt->dev->pdev, |
232 | ppgtt->pt_dma_addr[i], |
232 | ppgtt->pt_dma_addr[i], |
233 | 4096, PCI_DMA_BIDIRECTIONAL); |
233 | 4096, PCI_DMA_BIDIRECTIONAL); |
234 | } |
234 | } |
235 | 235 | ||
236 | kfree(ppgtt->pt_dma_addr); |
236 | kfree(ppgtt->pt_dma_addr); |
237 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
237 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
238 | __free_page(ppgtt->pt_pages[i]); |
238 | __free_page(ppgtt->pt_pages[i]); |
239 | kfree(ppgtt->pt_pages); |
239 | kfree(ppgtt->pt_pages); |
240 | kfree(ppgtt); |
240 | kfree(ppgtt); |
241 | } |
241 | } |
242 | 242 | ||
243 | static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
243 | static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
244 | { |
244 | { |
245 | struct drm_device *dev = ppgtt->dev; |
245 | struct drm_device *dev = ppgtt->dev; |
246 | struct drm_i915_private *dev_priv = dev->dev_private; |
246 | struct drm_i915_private *dev_priv = dev->dev_private; |
247 | unsigned first_pd_entry_in_global_pt; |
247 | unsigned first_pd_entry_in_global_pt; |
248 | int i; |
248 | int i; |
249 | int ret = -ENOMEM; |
249 | int ret = -ENOMEM; |
250 | 250 | ||
251 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
251 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
252 | * entries. For aliasing ppgtt support we just steal them at the end for |
252 | * entries. For aliasing ppgtt support we just steal them at the end for |
253 | * now. */ |
253 | * now. */ |
254 | first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); |
254 | first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); |
255 | 255 | ||
256 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
256 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
257 | ppgtt->enable = gen6_ppgtt_enable; |
257 | ppgtt->enable = gen6_ppgtt_enable; |
258 | ppgtt->clear_range = gen6_ppgtt_clear_range; |
258 | ppgtt->clear_range = gen6_ppgtt_clear_range; |
259 | ppgtt->insert_entries = gen6_ppgtt_insert_entries; |
259 | ppgtt->insert_entries = gen6_ppgtt_insert_entries; |
260 | ppgtt->cleanup = gen6_ppgtt_cleanup; |
260 | ppgtt->cleanup = gen6_ppgtt_cleanup; |
261 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
261 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
262 | GFP_KERNEL); |
262 | GFP_KERNEL); |
263 | if (!ppgtt->pt_pages) |
263 | if (!ppgtt->pt_pages) |
264 | return -ENOMEM; |
264 | return -ENOMEM; |
265 | 265 | ||
266 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
266 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
267 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
267 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
268 | if (!ppgtt->pt_pages[i]) |
268 | if (!ppgtt->pt_pages[i]) |
269 | goto err_pt_alloc; |
269 | goto err_pt_alloc; |
270 | } |
270 | } |
271 | 271 | ||
272 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries, |
272 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries, |
273 | GFP_KERNEL); |
273 | GFP_KERNEL); |
274 | if (!ppgtt->pt_dma_addr) |
274 | if (!ppgtt->pt_dma_addr) |
275 | goto err_pt_alloc; |
275 | goto err_pt_alloc; |
276 | 276 | ||
277 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
277 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
278 | dma_addr_t pt_addr; |
278 | dma_addr_t pt_addr; |
279 | 279 | ||
280 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
280 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
281 | PCI_DMA_BIDIRECTIONAL); |
281 | PCI_DMA_BIDIRECTIONAL); |
282 | 282 | ||
283 | ppgtt->pt_dma_addr[i] = pt_addr; |
283 | ppgtt->pt_dma_addr[i] = pt_addr; |
284 | } |
284 | } |
285 | 285 | ||
286 | ppgtt->clear_range(ppgtt, 0, |
286 | ppgtt->clear_range(ppgtt, 0, |
287 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
287 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
288 | 288 | ||
289 | ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); |
289 | ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); |
290 | 290 | ||
291 | return 0; |
291 | return 0; |
292 | 292 | ||
293 | err_pd_pin: |
293 | err_pd_pin: |
294 | if (ppgtt->pt_dma_addr) { |
294 | if (ppgtt->pt_dma_addr) { |
295 | for (i--; i >= 0; i--) |
295 | for (i--; i >= 0; i--) |
296 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
296 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
297 | 4096, PCI_DMA_BIDIRECTIONAL); |
297 | 4096, PCI_DMA_BIDIRECTIONAL); |
298 | } |
298 | } |
299 | err_pt_alloc: |
299 | err_pt_alloc: |
300 | kfree(ppgtt->pt_dma_addr); |
300 | kfree(ppgtt->pt_dma_addr); |
301 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
301 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
302 | if (ppgtt->pt_pages[i]) |
302 | if (ppgtt->pt_pages[i]) |
303 | __free_page(ppgtt->pt_pages[i]); |
303 | __free_page(ppgtt->pt_pages[i]); |
304 | } |
304 | } |
305 | kfree(ppgtt->pt_pages); |
305 | kfree(ppgtt->pt_pages); |
306 | 306 | ||
307 | return ret; |
307 | return ret; |
308 | } |
308 | } |
309 | 309 | ||
310 | static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
310 | static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
311 | { |
311 | { |
312 | struct drm_i915_private *dev_priv = dev->dev_private; |
312 | struct drm_i915_private *dev_priv = dev->dev_private; |
313 | struct i915_hw_ppgtt *ppgtt; |
313 | struct i915_hw_ppgtt *ppgtt; |
314 | int ret; |
314 | int ret; |
315 | 315 | ||
316 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
316 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
317 | if (!ppgtt) |
317 | if (!ppgtt) |
318 | return -ENOMEM; |
318 | return -ENOMEM; |
319 | 319 | ||
320 | ppgtt->dev = dev; |
320 | ppgtt->dev = dev; |
321 | ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; |
321 | ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; |
322 | 322 | ||
323 | if (INTEL_INFO(dev)->gen < 8) |
323 | if (INTEL_INFO(dev)->gen < 8) |
324 | ret = gen6_ppgtt_init(ppgtt); |
324 | ret = gen6_ppgtt_init(ppgtt); |
325 | else |
325 | else |
326 | BUG(); |
326 | BUG(); |
327 | 327 | ||
328 | if (ret) |
328 | if (ret) |
329 | kfree(ppgtt); |
329 | kfree(ppgtt); |
330 | else |
330 | else |
331 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
331 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
332 | 332 | ||
333 | return ret; |
333 | return ret; |
334 | } |
334 | } |
335 | 335 | ||
336 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
336 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
337 | { |
337 | { |
338 | struct drm_i915_private *dev_priv = dev->dev_private; |
338 | struct drm_i915_private *dev_priv = dev->dev_private; |
339 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
339 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
340 | 340 | ||
341 | if (!ppgtt) |
341 | if (!ppgtt) |
342 | return; |
342 | return; |
343 | 343 | ||
344 | ppgtt->cleanup(ppgtt); |
344 | ppgtt->cleanup(ppgtt); |
345 | dev_priv->mm.aliasing_ppgtt = NULL; |
345 | dev_priv->mm.aliasing_ppgtt = NULL; |
346 | } |
346 | } |
347 | 347 | ||
348 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
348 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
349 | struct drm_i915_gem_object *obj, |
349 | struct drm_i915_gem_object *obj, |
350 | enum i915_cache_level cache_level) |
350 | enum i915_cache_level cache_level) |
351 | { |
351 | { |
352 | ppgtt->insert_entries(ppgtt, obj->pages, |
352 | ppgtt->insert_entries(ppgtt, obj->pages, |
353 | obj->gtt_space->start >> PAGE_SHIFT, |
353 | obj->gtt_space->start >> PAGE_SHIFT, |
354 | cache_level); |
354 | cache_level); |
355 | } |
355 | } |
356 | 356 | ||
357 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
357 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
358 | struct drm_i915_gem_object *obj) |
358 | struct drm_i915_gem_object *obj) |
359 | { |
359 | { |
360 | ppgtt->clear_range(ppgtt, |
360 | ppgtt->clear_range(ppgtt, |
361 | obj->gtt_space->start >> PAGE_SHIFT, |
361 | obj->gtt_space->start >> PAGE_SHIFT, |
362 | obj->base.size >> PAGE_SHIFT); |
362 | obj->base.size >> PAGE_SHIFT); |
363 | } |
363 | } |
364 | 364 | ||
365 | extern int intel_iommu_gfx_mapped; |
365 | extern int intel_iommu_gfx_mapped; |
366 | /* Certain Gen5 chipsets require require idling the GPU before |
366 | /* Certain Gen5 chipsets require require idling the GPU before |
367 | * unmapping anything from the GTT when VT-d is enabled. |
367 | * unmapping anything from the GTT when VT-d is enabled. |
368 | */ |
368 | */ |
369 | static inline bool needs_idle_maps(struct drm_device *dev) |
369 | static inline bool needs_idle_maps(struct drm_device *dev) |
370 | { |
370 | { |
371 | #ifdef CONFIG_INTEL_IOMMU |
371 | #ifdef CONFIG_INTEL_IOMMU |
372 | /* Query intel_iommu to see if we need the workaround. Presumably that |
372 | /* Query intel_iommu to see if we need the workaround. Presumably that |
373 | * was loaded first. |
373 | * was loaded first. |
374 | */ |
374 | */ |
375 | if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) |
375 | if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) |
376 | return true; |
376 | return true; |
377 | #endif |
377 | #endif |
378 | return false; |
378 | return false; |
379 | } |
379 | } |
380 | 380 | ||
381 | static bool do_idling(struct drm_i915_private *dev_priv) |
381 | static bool do_idling(struct drm_i915_private *dev_priv) |
382 | { |
382 | { |
383 | bool ret = dev_priv->mm.interruptible; |
383 | bool ret = dev_priv->mm.interruptible; |
384 | 384 | ||
385 | if (unlikely(dev_priv->gtt.do_idle_maps)) { |
385 | if (unlikely(dev_priv->gtt.do_idle_maps)) { |
386 | dev_priv->mm.interruptible = false; |
386 | dev_priv->mm.interruptible = false; |
387 | if (i915_gpu_idle(dev_priv->dev)) { |
387 | if (i915_gpu_idle(dev_priv->dev)) { |
388 | DRM_ERROR("Couldn't idle GPU\n"); |
388 | DRM_ERROR("Couldn't idle GPU\n"); |
389 | /* Wait a bit, in hopes it avoids the hang */ |
389 | /* Wait a bit, in hopes it avoids the hang */ |
390 | udelay(10); |
390 | udelay(10); |
391 | } |
391 | } |
392 | } |
392 | } |
393 | 393 | ||
394 | return ret; |
394 | return ret; |
395 | } |
395 | } |
396 | 396 | ||
397 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
397 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
398 | { |
398 | { |
399 | if (unlikely(dev_priv->gtt.do_idle_maps)) |
399 | if (unlikely(dev_priv->gtt.do_idle_maps)) |
400 | dev_priv->mm.interruptible = interruptible; |
400 | dev_priv->mm.interruptible = interruptible; |
401 | } |
401 | } |
402 | 402 | ||
403 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
403 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
404 | { |
404 | { |
405 | struct drm_i915_private *dev_priv = dev->dev_private; |
405 | struct drm_i915_private *dev_priv = dev->dev_private; |
406 | struct drm_i915_gem_object *obj; |
406 | struct drm_i915_gem_object *obj; |
407 | 407 | ||
408 | /* First fill our portion of the GTT with scratch pages */ |
408 | /* First fill our portion of the GTT with scratch pages */ |
409 | dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE, |
409 | dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE, |
410 | dev_priv->gtt.total / PAGE_SIZE); |
410 | dev_priv->gtt.total / PAGE_SIZE); |
411 | 411 | ||
412 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
412 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
413 | i915_gem_clflush_object(obj); |
413 | i915_gem_clflush_object(obj); |
414 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
414 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
415 | } |
415 | } |
416 | 416 | ||
417 | i915_gem_chipset_flush(dev); |
417 | i915_gem_chipset_flush(dev); |
418 | } |
418 | } |
419 | 419 | ||
420 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
420 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
421 | { |
421 | { |
422 | if (obj->has_dma_mapping) |
422 | if (obj->has_dma_mapping) |
423 | return 0; |
423 | return 0; |
424 | 424 | ||
425 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
425 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
426 | obj->pages->sgl, obj->pages->nents, |
426 | obj->pages->sgl, obj->pages->nents, |
427 | PCI_DMA_BIDIRECTIONAL)) |
427 | PCI_DMA_BIDIRECTIONAL)) |
428 | return -ENOSPC; |
428 | return -ENOSPC; |
429 | 429 | ||
430 | return 0; |
430 | return 0; |
431 | } |
431 | } |
432 | 432 | ||
433 | /* |
433 | /* |
434 | * Binds an object into the global gtt with the specified cache level. The object |
434 | * Binds an object into the global gtt with the specified cache level. The object |
435 | * will be accessible to the GPU via commands whose operands reference offsets |
435 | * will be accessible to the GPU via commands whose operands reference offsets |
436 | * within the global GTT as well as accessible by the GPU through the GMADR |
436 | * within the global GTT as well as accessible by the GPU through the GMADR |
437 | * mapped BAR (dev_priv->mm.gtt->gtt). |
437 | * mapped BAR (dev_priv->mm.gtt->gtt). |
438 | */ |
438 | */ |
439 | static void gen6_ggtt_insert_entries(struct drm_device *dev, |
439 | static void gen6_ggtt_insert_entries(struct drm_device *dev, |
440 | struct sg_table *st, |
440 | struct sg_table *st, |
441 | unsigned int first_entry, |
441 | unsigned int first_entry, |
442 | enum i915_cache_level level) |
442 | enum i915_cache_level level) |
443 | { |
443 | { |
444 | struct drm_i915_private *dev_priv = dev->dev_private; |
444 | struct drm_i915_private *dev_priv = dev->dev_private; |
445 | gen6_gtt_pte_t __iomem *gtt_entries = |
445 | gen6_gtt_pte_t __iomem *gtt_entries = |
446 | (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
446 | (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
447 | int i = 0; |
447 | int i = 0; |
448 | struct sg_page_iter sg_iter; |
448 | struct sg_page_iter sg_iter; |
449 | dma_addr_t addr; |
449 | dma_addr_t addr; |
450 | 450 | ||
451 | for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { |
451 | for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { |
452 | addr = sg_page_iter_dma_address(&sg_iter); |
452 | addr = sg_page_iter_dma_address(&sg_iter); |
453 | iowrite32(gen6_pte_encode(dev, addr, level), >t_entries[i]); |
453 | iowrite32(gen6_pte_encode(dev, addr, level), >t_entries[i]); |
454 | i++; |
454 | i++; |
455 | } |
455 | } |
456 | 456 | ||
457 | /* XXX: This serves as a posting read to make sure that the PTE has |
457 | /* XXX: This serves as a posting read to make sure that the PTE has |
458 | * actually been updated. There is some concern that even though |
458 | * actually been updated. There is some concern that even though |
459 | * registers and PTEs are within the same BAR that they are potentially |
459 | * registers and PTEs are within the same BAR that they are potentially |
460 | * of NUMA access patterns. Therefore, even with the way we assume |
460 | * of NUMA access patterns. Therefore, even with the way we assume |
461 | * hardware should work, we must keep this posting read for paranoia. |
461 | * hardware should work, we must keep this posting read for paranoia. |
462 | */ |
462 | */ |
463 | if (i != 0) |
463 | if (i != 0) |
464 | WARN_ON(readl(>t_entries[i-1]) |
464 | WARN_ON(readl(>t_entries[i-1]) |
465 | != gen6_pte_encode(dev, addr, level)); |
465 | != gen6_pte_encode(dev, addr, level)); |
466 | 466 | ||
467 | /* This next bit makes the above posting read even more important. We |
467 | /* This next bit makes the above posting read even more important. We |
468 | * want to flush the TLBs only after we're certain all the PTE updates |
468 | * want to flush the TLBs only after we're certain all the PTE updates |
469 | * have finished. |
469 | * have finished. |
470 | */ |
470 | */ |
471 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
471 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
472 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
472 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
473 | } |
473 | } |
474 | 474 | ||
475 | static void gen6_ggtt_clear_range(struct drm_device *dev, |
475 | static void gen6_ggtt_clear_range(struct drm_device *dev, |
476 | unsigned int first_entry, |
476 | unsigned int first_entry, |
477 | unsigned int num_entries) |
477 | unsigned int num_entries) |
478 | { |
478 | { |
479 | struct drm_i915_private *dev_priv = dev->dev_private; |
479 | struct drm_i915_private *dev_priv = dev->dev_private; |
480 | gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
480 | gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
481 | (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
481 | (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
482 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
482 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
483 | int i; |
483 | int i; |
484 | 484 | ||
485 | // if (WARN(num_entries > max_entries, |
485 | // if (WARN(num_entries > max_entries, |
486 | // "First entry = %d; Num entries = %d (max=%d)\n", |
486 | // "First entry = %d; Num entries = %d (max=%d)\n", |
487 | // first_entry, num_entries, max_entries)) |
487 | // first_entry, num_entries, max_entries)) |
488 | if (num_entries > max_entries) |
488 | if (num_entries > max_entries) |
489 | num_entries = max_entries; |
489 | num_entries = max_entries; |
490 | 490 | ||
491 | scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma, |
491 | scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma, |
492 | I915_CACHE_LLC); |
492 | I915_CACHE_LLC); |
493 | for (i = 0; i < num_entries; i++) |
493 | for (i = 0; i < num_entries; i++) |
494 | iowrite32(scratch_pte, >t_base[i]); |
494 | iowrite32(scratch_pte, >t_base[i]); |
495 | readl(gtt_base); |
495 | readl(gtt_base); |
496 | } |
496 | } |
497 | 497 | ||
498 | 498 | ||
499 | static void i915_ggtt_insert_entries(struct drm_device *dev, |
499 | static void i915_ggtt_insert_entries(struct drm_device *dev, |
500 | struct sg_table *st, |
500 | struct sg_table *st, |
501 | unsigned int pg_start, |
501 | unsigned int pg_start, |
502 | enum i915_cache_level cache_level) |
502 | enum i915_cache_level cache_level) |
503 | { |
503 | { |
504 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
504 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
505 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
505 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
506 | 506 | ||
507 | intel_gtt_insert_sg_entries(st, pg_start, flags); |
507 | intel_gtt_insert_sg_entries(st, pg_start, flags); |
508 | 508 | ||
509 | } |
509 | } |
510 | 510 | ||
511 | static void i915_ggtt_clear_range(struct drm_device *dev, |
511 | static void i915_ggtt_clear_range(struct drm_device *dev, |
512 | unsigned int first_entry, |
512 | unsigned int first_entry, |
513 | unsigned int num_entries) |
513 | unsigned int num_entries) |
514 | { |
514 | { |
515 | intel_gtt_clear_range(first_entry, num_entries); |
515 | intel_gtt_clear_range(first_entry, num_entries); |
516 | } |
516 | } |
517 | 517 | ||
518 | 518 | ||
519 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
519 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
520 | enum i915_cache_level cache_level) |
520 | enum i915_cache_level cache_level) |
521 | { |
521 | { |
522 | struct drm_device *dev = obj->base.dev; |
522 | struct drm_device *dev = obj->base.dev; |
523 | struct drm_i915_private *dev_priv = dev->dev_private; |
523 | struct drm_i915_private *dev_priv = dev->dev_private; |
524 | 524 | ||
525 | dev_priv->gtt.gtt_insert_entries(dev, obj->pages, |
525 | dev_priv->gtt.gtt_insert_entries(dev, obj->pages, |
526 | obj->gtt_space->start >> PAGE_SHIFT, |
526 | obj->gtt_space->start >> PAGE_SHIFT, |
527 | cache_level); |
527 | cache_level); |
528 | 528 | ||
529 | obj->has_global_gtt_mapping = 1; |
529 | obj->has_global_gtt_mapping = 1; |
530 | } |
530 | } |
531 | 531 | ||
532 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
532 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
533 | { |
533 | { |
534 | struct drm_device *dev = obj->base.dev; |
534 | struct drm_device *dev = obj->base.dev; |
535 | struct drm_i915_private *dev_priv = dev->dev_private; |
535 | struct drm_i915_private *dev_priv = dev->dev_private; |
536 | 536 | ||
537 | dev_priv->gtt.gtt_clear_range(obj->base.dev, |
537 | dev_priv->gtt.gtt_clear_range(obj->base.dev, |
538 | obj->gtt_space->start >> PAGE_SHIFT, |
538 | obj->gtt_space->start >> PAGE_SHIFT, |
539 | obj->base.size >> PAGE_SHIFT); |
539 | obj->base.size >> PAGE_SHIFT); |
540 | 540 | ||
541 | obj->has_global_gtt_mapping = 0; |
541 | obj->has_global_gtt_mapping = 0; |
542 | } |
542 | } |
543 | 543 | ||
544 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
544 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
545 | { |
545 | { |
546 | struct drm_device *dev = obj->base.dev; |
546 | struct drm_device *dev = obj->base.dev; |
547 | struct drm_i915_private *dev_priv = dev->dev_private; |
547 | struct drm_i915_private *dev_priv = dev->dev_private; |
548 | bool interruptible; |
548 | bool interruptible; |
549 | 549 | ||
550 | interruptible = do_idling(dev_priv); |
550 | interruptible = do_idling(dev_priv); |
551 | 551 | ||
552 | if (!obj->has_dma_mapping) |
552 | if (!obj->has_dma_mapping) |
553 | dma_unmap_sg(&dev->pdev->dev, |
553 | dma_unmap_sg(&dev->pdev->dev, |
554 | obj->pages->sgl, obj->pages->nents, |
554 | obj->pages->sgl, obj->pages->nents, |
555 | PCI_DMA_BIDIRECTIONAL); |
555 | PCI_DMA_BIDIRECTIONAL); |
556 | 556 | ||
557 | undo_idling(dev_priv, interruptible); |
557 | undo_idling(dev_priv, interruptible); |
558 | } |
558 | } |
559 | 559 | ||
560 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
560 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
561 | unsigned long color, |
561 | unsigned long color, |
562 | unsigned long *start, |
562 | unsigned long *start, |
563 | unsigned long *end) |
563 | unsigned long *end) |
564 | { |
564 | { |
565 | if (node->color != color) |
565 | if (node->color != color) |
566 | *start += 4096; |
566 | *start += 4096; |
567 | 567 | ||
568 | if (!list_empty(&node->node_list)) { |
568 | if (!list_empty(&node->node_list)) { |
569 | node = list_entry(node->node_list.next, |
569 | node = list_entry(node->node_list.next, |
570 | struct drm_mm_node, |
570 | struct drm_mm_node, |
571 | node_list); |
571 | node_list); |
572 | if (node->allocated && node->color != color) |
572 | if (node->allocated && node->color != color) |
573 | *end -= 4096; |
573 | *end -= 4096; |
574 | } |
574 | } |
575 | } |
575 | } |
576 | void i915_gem_setup_global_gtt(struct drm_device *dev, |
576 | void i915_gem_setup_global_gtt(struct drm_device *dev, |
577 | unsigned long start, |
577 | unsigned long start, |
578 | unsigned long mappable_end, |
578 | unsigned long mappable_end, |
579 | unsigned long end) |
579 | unsigned long end) |
580 | { |
580 | { |
581 | /* Let GEM Manage all of the aperture. |
581 | /* Let GEM Manage all of the aperture. |
582 | * |
582 | * |
583 | * However, leave one page at the end still bound to the scratch page. |
583 | * However, leave one page at the end still bound to the scratch page. |
584 | * There are a number of places where the hardware apparently prefetches |
584 | * There are a number of places where the hardware apparently prefetches |
585 | * past the end of the object, and we've seen multiple hangs with the |
585 | * past the end of the object, and we've seen multiple hangs with the |
586 | * GPU head pointer stuck in a batchbuffer bound at the last page of the |
586 | * GPU head pointer stuck in a batchbuffer bound at the last page of the |
587 | * aperture. One page should be enough to keep any prefetching inside |
587 | * aperture. One page should be enough to keep any prefetching inside |
588 | * of the aperture. |
588 | * of the aperture. |
589 | */ |
589 | */ |
590 | drm_i915_private_t *dev_priv = dev->dev_private; |
590 | drm_i915_private_t *dev_priv = dev->dev_private; |
591 | struct drm_mm_node *entry; |
591 | struct drm_mm_node *entry; |
592 | struct drm_i915_gem_object *obj; |
592 | struct drm_i915_gem_object *obj; |
593 | unsigned long hole_start, hole_end; |
593 | unsigned long hole_start, hole_end; |
594 | 594 | ||
595 | BUG_ON(mappable_end > end); |
595 | BUG_ON(mappable_end > end); |
596 | 596 | ||
597 | /* Subtract the guard page ... */ |
597 | /* Subtract the guard page ... */ |
598 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); |
598 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); |
599 | if (!HAS_LLC(dev)) |
599 | if (!HAS_LLC(dev)) |
600 | dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; |
600 | dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; |
601 | 601 | ||
602 | /* Mark any preallocated objects as occupied */ |
602 | /* Mark any preallocated objects as occupied */ |
603 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
603 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
604 | DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n", |
604 | DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n", |
605 | obj->gtt_offset, obj->base.size); |
605 | obj->gtt_offset, obj->base.size); |
606 | 606 | ||
607 | BUG_ON(obj->gtt_space != I915_GTT_RESERVED); |
607 | BUG_ON(obj->gtt_space != I915_GTT_RESERVED); |
608 | obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space, |
608 | obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space, |
609 | obj->gtt_offset, |
609 | obj->gtt_offset, |
610 | obj->base.size, |
610 | obj->base.size, |
611 | false); |
611 | false); |
612 | obj->has_global_gtt_mapping = 1; |
612 | obj->has_global_gtt_mapping = 1; |
613 | } |
613 | } |
614 | 614 | ||
615 | dev_priv->gtt.start = start; |
615 | dev_priv->gtt.start = start; |
616 | dev_priv->gtt.total = end - start; |
616 | dev_priv->gtt.total = end - start; |
617 | 617 | ||
618 | /* Clear any non-preallocated blocks */ |
618 | /* Clear any non-preallocated blocks */ |
619 | drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space, |
619 | drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space, |
620 | hole_start, hole_end) { |
620 | hole_start, hole_end) { |
621 | DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", |
621 | DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", |
622 | hole_start, hole_end); |
622 | hole_start, hole_end); |
623 | dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE, |
623 | dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE, |
624 | (hole_end-hole_start) / PAGE_SIZE); |
624 | (hole_end-hole_start) / PAGE_SIZE); |
625 | } |
625 | } |
626 | 626 | ||
627 | /* And finally clear the reserved guard page */ |
627 | /* And finally clear the reserved guard page */ |
628 | dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1); |
628 | dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1); |
629 | } |
629 | } |
630 | 630 | ||
631 | static bool |
631 | static bool |
632 | intel_enable_ppgtt(struct drm_device *dev) |
632 | intel_enable_ppgtt(struct drm_device *dev) |
633 | { |
633 | { |
634 | if (i915_enable_ppgtt >= 0) |
634 | if (i915_enable_ppgtt >= 0) |
635 | return i915_enable_ppgtt; |
635 | return i915_enable_ppgtt; |
636 | 636 | ||
637 | #ifdef CONFIG_INTEL_IOMMU |
637 | #ifdef CONFIG_INTEL_IOMMU |
638 | /* Disable ppgtt on SNB if VT-d is on. */ |
638 | /* Disable ppgtt on SNB if VT-d is on. */ |
639 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
639 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
640 | return false; |
640 | return false; |
641 | #endif |
641 | #endif |
642 | 642 | ||
643 | return true; |
643 | return true; |
644 | } |
644 | } |
645 | 645 | ||
646 | void i915_gem_init_global_gtt(struct drm_device *dev) |
646 | void i915_gem_init_global_gtt(struct drm_device *dev) |
647 | { |
647 | { |
648 | struct drm_i915_private *dev_priv = dev->dev_private; |
648 | struct drm_i915_private *dev_priv = dev->dev_private; |
649 | unsigned long gtt_size, mappable_size; |
649 | unsigned long gtt_size, mappable_size; |
650 | 650 | ||
651 | gtt_size = dev_priv->gtt.total; |
651 | gtt_size = dev_priv->gtt.total; |
652 | mappable_size = dev_priv->gtt.mappable_end; |
652 | mappable_size = dev_priv->gtt.mappable_end; |
653 | 653 | ||
654 | #if 0 |
654 | #if 0 |
655 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
655 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
656 | int ret; |
656 | int ret; |
657 | 657 | ||
658 | if (INTEL_INFO(dev)->gen <= 7) { |
658 | if (INTEL_INFO(dev)->gen <= 7) { |
659 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
659 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
660 | * aperture accordingly when using aliasing ppgtt. */ |
660 | * aperture accordingly when using aliasing ppgtt. */ |
661 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
661 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
662 | } |
662 | } |
663 | - | ||
664 | // gtt_size -= LFB_SIZE; |
- | |
665 | 663 | ||
666 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
664 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
667 | 665 | ||
668 | ret = i915_gem_init_aliasing_ppgtt(dev); |
666 | ret = i915_gem_init_aliasing_ppgtt(dev); |
669 | if (!ret) |
667 | if (!ret) |
670 | return; |
668 | return; |
671 | 669 | ||
672 | DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); |
670 | DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); |
673 | drm_mm_takedown(&dev_priv->mm.gtt_space); |
671 | drm_mm_takedown(&dev_priv->mm.gtt_space); |
674 | gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
672 | gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
675 | } |
673 | } |
676 | #endif |
674 | #endif |
677 | 675 | ||
678 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
676 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
679 | } |
677 | } |
680 | 678 | ||
681 | static int setup_scratch_page(struct drm_device *dev) |
679 | static int setup_scratch_page(struct drm_device *dev) |
682 | { |
680 | { |
683 | struct drm_i915_private *dev_priv = dev->dev_private; |
681 | struct drm_i915_private *dev_priv = dev->dev_private; |
684 | struct page *page; |
682 | struct page *page; |
685 | dma_addr_t dma_addr; |
683 | dma_addr_t dma_addr; |
686 | 684 | ||
687 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
685 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
688 | if (page == NULL) |
686 | if (page == NULL) |
689 | return -ENOMEM; |
687 | return -ENOMEM; |
690 | get_page(page); |
688 | get_page(page); |
691 | set_pages_uc(page, 1); |
689 | set_pages_uc(page, 1); |
692 | 690 | ||
693 | #ifdef CONFIG_INTEL_IOMMU |
691 | #ifdef CONFIG_INTEL_IOMMU |
694 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
692 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
695 | PCI_DMA_BIDIRECTIONAL); |
693 | PCI_DMA_BIDIRECTIONAL); |
696 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
694 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
697 | return -EINVAL; |
695 | return -EINVAL; |
698 | #else |
696 | #else |
699 | dma_addr = page_to_phys(page); |
697 | dma_addr = page_to_phys(page); |
700 | #endif |
698 | #endif |
701 | dev_priv->gtt.scratch_page = page; |
699 | dev_priv->gtt.scratch_page = page; |
702 | dev_priv->gtt.scratch_page_dma = dma_addr; |
700 | dev_priv->gtt.scratch_page_dma = dma_addr; |
703 | 701 | ||
704 | return 0; |
702 | return 0; |
705 | } |
703 | } |
706 | 704 | ||
707 | static void teardown_scratch_page(struct drm_device *dev) |
705 | static void teardown_scratch_page(struct drm_device *dev) |
708 | { |
706 | { |
709 | struct drm_i915_private *dev_priv = dev->dev_private; |
707 | struct drm_i915_private *dev_priv = dev->dev_private; |
710 | set_pages_wb(dev_priv->gtt.scratch_page, 1); |
708 | set_pages_wb(dev_priv->gtt.scratch_page, 1); |
711 | pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma, |
709 | pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma, |
712 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
710 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
713 | put_page(dev_priv->gtt.scratch_page); |
711 | put_page(dev_priv->gtt.scratch_page); |
714 | __free_page(dev_priv->gtt.scratch_page); |
712 | __free_page(dev_priv->gtt.scratch_page); |
715 | } |
713 | } |
716 | 714 | ||
717 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
715 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
718 | { |
716 | { |
719 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
717 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
720 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
718 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
721 | return snb_gmch_ctl << 20; |
719 | return snb_gmch_ctl << 20; |
722 | } |
720 | } |
723 | 721 | ||
724 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
722 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
725 | { |
723 | { |
726 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
724 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
727 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
725 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
728 | return snb_gmch_ctl << 25; /* 32 MB units */ |
726 | return snb_gmch_ctl << 25; /* 32 MB units */ |
729 | } |
727 | } |
730 | 728 | ||
731 | static int gen6_gmch_probe(struct drm_device *dev, |
729 | static int gen6_gmch_probe(struct drm_device *dev, |
732 | size_t *gtt_total, |
730 | size_t *gtt_total, |
733 | size_t *stolen, |
731 | size_t *stolen, |
734 | phys_addr_t *mappable_base, |
732 | phys_addr_t *mappable_base, |
735 | unsigned long *mappable_end) |
733 | unsigned long *mappable_end) |
736 | { |
734 | { |
737 | struct drm_i915_private *dev_priv = dev->dev_private; |
735 | struct drm_i915_private *dev_priv = dev->dev_private; |
738 | phys_addr_t gtt_bus_addr; |
736 | phys_addr_t gtt_bus_addr; |
739 | unsigned int gtt_size; |
737 | unsigned int gtt_size; |
740 | u16 snb_gmch_ctl; |
738 | u16 snb_gmch_ctl; |
741 | int ret; |
739 | int ret; |
742 | 740 | ||
743 | *mappable_base = pci_resource_start(dev->pdev, 2); |
741 | *mappable_base = pci_resource_start(dev->pdev, 2); |
744 | *mappable_end = pci_resource_len(dev->pdev, 2); |
742 | *mappable_end = pci_resource_len(dev->pdev, 2); |
745 | 743 | ||
746 | /* 64/512MB is the current min/max we actually know of, but this is just |
744 | /* 64/512MB is the current min/max we actually know of, but this is just |
747 | * a coarse sanity check. |
745 | * a coarse sanity check. |
748 | */ |
746 | */ |
749 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
747 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
750 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
748 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
751 | dev_priv->gtt.mappable_end); |
749 | dev_priv->gtt.mappable_end); |
752 | return -ENXIO; |
750 | return -ENXIO; |
753 | } |
751 | } |
754 | 752 | ||
755 | if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) |
753 | if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) |
756 | pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); |
754 | pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); |
757 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
755 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
758 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
756 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
759 | 757 | ||
760 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
758 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
761 | *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
759 | *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
762 | 760 | ||
763 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
761 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
764 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + |
762 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + |
765 | (pci_resource_len(dev->pdev, 0) / 2); |
763 | (pci_resource_len(dev->pdev, 0) / 2); |
766 | 764 | ||
767 | dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
765 | dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
768 | if (!dev_priv->gtt.gsm) { |
766 | if (!dev_priv->gtt.gsm) { |
769 | DRM_ERROR("Failed to map the gtt page table\n"); |
767 | DRM_ERROR("Failed to map the gtt page table\n"); |
770 | return -ENOMEM; |
768 | return -ENOMEM; |
771 | } |
769 | } |
772 | 770 | ||
773 | ret = setup_scratch_page(dev); |
771 | ret = setup_scratch_page(dev); |
774 | if (ret) |
772 | if (ret) |
775 | DRM_ERROR("Scratch setup failed\n"); |
773 | DRM_ERROR("Scratch setup failed\n"); |
776 | 774 | ||
777 | dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range; |
775 | dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range; |
778 | dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries; |
776 | dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries; |
779 | 777 | ||
780 | return ret; |
778 | return ret; |
781 | } |
779 | } |
782 | 780 | ||
783 | static void gen6_gmch_remove(struct drm_device *dev) |
781 | static void gen6_gmch_remove(struct drm_device *dev) |
784 | { |
782 | { |
785 | struct drm_i915_private *dev_priv = dev->dev_private; |
783 | struct drm_i915_private *dev_priv = dev->dev_private; |
786 | iounmap(dev_priv->gtt.gsm); |
784 | iounmap(dev_priv->gtt.gsm); |
787 | teardown_scratch_page(dev_priv->dev); |
785 | teardown_scratch_page(dev_priv->dev); |
788 | } |
786 | } |
789 | 787 | ||
790 | static int i915_gmch_probe(struct drm_device *dev, |
788 | static int i915_gmch_probe(struct drm_device *dev, |
791 | size_t *gtt_total, |
789 | size_t *gtt_total, |
792 | size_t *stolen, |
790 | size_t *stolen, |
793 | phys_addr_t *mappable_base, |
791 | phys_addr_t *mappable_base, |
794 | unsigned long *mappable_end) |
792 | unsigned long *mappable_end) |
795 | { |
793 | { |
796 | struct drm_i915_private *dev_priv = dev->dev_private; |
794 | struct drm_i915_private *dev_priv = dev->dev_private; |
797 | int ret; |
795 | int ret; |
798 | 796 | ||
799 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); |
797 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); |
800 | if (!ret) { |
798 | if (!ret) { |
801 | DRM_ERROR("failed to set up gmch\n"); |
799 | DRM_ERROR("failed to set up gmch\n"); |
802 | return -EIO; |
800 | return -EIO; |
803 | } |
801 | } |
804 | 802 | ||
805 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
803 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
806 | 804 | ||
807 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
805 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
808 | dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range; |
806 | dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range; |
809 | dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries; |
807 | dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries; |
810 | 808 | ||
811 | return 0; |
809 | return 0; |
812 | } |
810 | } |
813 | 811 | ||
814 | static void i915_gmch_remove(struct drm_device *dev) |
812 | static void i915_gmch_remove(struct drm_device *dev) |
815 | { |
813 | { |
816 | // intel_gmch_remove(); |
814 | // intel_gmch_remove(); |
817 | } |
815 | } |
818 | 816 | ||
819 | int i915_gem_gtt_init(struct drm_device *dev) |
817 | int i915_gem_gtt_init(struct drm_device *dev) |
820 | { |
818 | { |
821 | struct drm_i915_private *dev_priv = dev->dev_private; |
819 | struct drm_i915_private *dev_priv = dev->dev_private; |
822 | struct i915_gtt *gtt = &dev_priv->gtt; |
820 | struct i915_gtt *gtt = &dev_priv->gtt; |
823 | int ret; |
821 | int ret; |
824 | 822 | ||
825 | if (INTEL_INFO(dev)->gen <= 5) { |
823 | if (INTEL_INFO(dev)->gen <= 5) { |
826 | dev_priv->gtt.gtt_probe = i915_gmch_probe; |
824 | dev_priv->gtt.gtt_probe = i915_gmch_probe; |
827 | dev_priv->gtt.gtt_remove = i915_gmch_remove; |
825 | dev_priv->gtt.gtt_remove = i915_gmch_remove; |
828 | } else { |
826 | } else { |
829 | dev_priv->gtt.gtt_probe = gen6_gmch_probe; |
827 | dev_priv->gtt.gtt_probe = gen6_gmch_probe; |
830 | dev_priv->gtt.gtt_remove = gen6_gmch_remove; |
828 | dev_priv->gtt.gtt_remove = gen6_gmch_remove; |
831 | } |
829 | } |
832 | 830 | ||
833 | ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total, |
831 | ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total, |
834 | &dev_priv->gtt.stolen_size, |
832 | &dev_priv->gtt.stolen_size, |
835 | >t->mappable_base, |
833 | >t->mappable_base, |
836 | >t->mappable_end); |
834 | >t->mappable_end); |
837 | if (ret) |
835 | if (ret) |
838 | return ret; |
836 | return ret; |
839 | 837 | ||
840 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
838 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
841 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
839 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
842 | dev_priv->gtt.total >> 20); |
840 | dev_priv->gtt.total >> 20); |
843 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", |
841 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", |
844 | dev_priv->gtt.mappable_end >> 20); |
842 | dev_priv->gtt.mappable_end >> 20); |
845 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", |
843 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", |
846 | dev_priv->gtt.stolen_size >> 20); |
844 | dev_priv->gtt.stolen_size >> 20); |
847 | 845 | ||
848 | return 0; |
846 | return 0; |
849 | } |
847 | } |
850 | 848 | ||
851 | struct scatterlist *sg_next(struct scatterlist *sg) |
849 | struct scatterlist *sg_next(struct scatterlist *sg) |
852 | { |
850 | { |
853 | if (sg_is_last(sg)) |
851 | if (sg_is_last(sg)) |
854 | return NULL; |
852 | return NULL; |
855 | 853 | ||
856 | sg++; |
854 | sg++; |
857 | if (unlikely(sg_is_chain(sg))) |
855 | if (unlikely(sg_is_chain(sg))) |
858 | sg = sg_chain_ptr(sg); |
856 | sg = sg_chain_ptr(sg); |
859 | 857 | ||
860 | return sg; |
858 | return sg; |
861 | } |
859 | } |
862 | 860 | ||
863 | 861 | ||
864 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
862 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
865 | sg_free_fn *free_fn) |
863 | sg_free_fn *free_fn) |
866 | { |
864 | { |
867 | struct scatterlist *sgl, *next; |
865 | struct scatterlist *sgl, *next; |
868 | 866 | ||
869 | if (unlikely(!table->sgl)) |
867 | if (unlikely(!table->sgl)) |
870 | return; |
868 | return; |
871 | 869 | ||
872 | sgl = table->sgl; |
870 | sgl = table->sgl; |
873 | while (table->orig_nents) { |
871 | while (table->orig_nents) { |
874 | unsigned int alloc_size = table->orig_nents; |
872 | unsigned int alloc_size = table->orig_nents; |
875 | unsigned int sg_size; |
873 | unsigned int sg_size; |
876 | 874 | ||
877 | /* |
875 | /* |
878 | * If we have more than max_ents segments left, |
876 | * If we have more than max_ents segments left, |
879 | * then assign 'next' to the sg table after the current one. |
877 | * then assign 'next' to the sg table after the current one. |
880 | * sg_size is then one less than alloc size, since the last |
878 | * sg_size is then one less than alloc size, since the last |
881 | * element is the chain pointer. |
879 | * element is the chain pointer. |
882 | */ |
880 | */ |
883 | if (alloc_size > max_ents) { |
881 | if (alloc_size > max_ents) { |
884 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
882 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
885 | alloc_size = max_ents; |
883 | alloc_size = max_ents; |
886 | sg_size = alloc_size - 1; |
884 | sg_size = alloc_size - 1; |
887 | } else { |
885 | } else { |
888 | sg_size = alloc_size; |
886 | sg_size = alloc_size; |
889 | next = NULL; |
887 | next = NULL; |
890 | } |
888 | } |
891 | 889 | ||
892 | table->orig_nents -= sg_size; |
890 | table->orig_nents -= sg_size; |
893 | kfree(sgl); |
891 | kfree(sgl); |
894 | sgl = next; |
892 | sgl = next; |
895 | } |
893 | } |
896 | 894 | ||
897 | table->sgl = NULL; |
895 | table->sgl = NULL; |
898 | } |
896 | } |
899 | 897 | ||
900 | void sg_free_table(struct sg_table *table) |
898 | void sg_free_table(struct sg_table *table) |
901 | { |
899 | { |
902 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
900 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
903 | } |
901 | } |
904 | 902 | ||
905 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
903 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
906 | { |
904 | { |
907 | struct scatterlist *sg, *prv; |
905 | struct scatterlist *sg, *prv; |
908 | unsigned int left; |
906 | unsigned int left; |
909 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
907 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
910 | 908 | ||
911 | #ifndef ARCH_HAS_SG_CHAIN |
909 | #ifndef ARCH_HAS_SG_CHAIN |
912 | BUG_ON(nents > max_ents); |
910 | BUG_ON(nents > max_ents); |
913 | #endif |
911 | #endif |
914 | 912 | ||
915 | memset(table, 0, sizeof(*table)); |
913 | memset(table, 0, sizeof(*table)); |
916 | 914 | ||
917 | left = nents; |
915 | left = nents; |
918 | prv = NULL; |
916 | prv = NULL; |
919 | do { |
917 | do { |
920 | unsigned int sg_size, alloc_size = left; |
918 | unsigned int sg_size, alloc_size = left; |
921 | 919 | ||
922 | if (alloc_size > max_ents) { |
920 | if (alloc_size > max_ents) { |
923 | alloc_size = max_ents; |
921 | alloc_size = max_ents; |
924 | sg_size = alloc_size - 1; |
922 | sg_size = alloc_size - 1; |
925 | } else |
923 | } else |
926 | sg_size = alloc_size; |
924 | sg_size = alloc_size; |
927 | 925 | ||
928 | left -= sg_size; |
926 | left -= sg_size; |
929 | 927 | ||
930 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
928 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
931 | if (unlikely(!sg)) { |
929 | if (unlikely(!sg)) { |
932 | /* |
930 | /* |
933 | * Adjust entry count to reflect that the last |
931 | * Adjust entry count to reflect that the last |
934 | * entry of the previous table won't be used for |
932 | * entry of the previous table won't be used for |
935 | * linkage. Without this, sg_kfree() may get |
933 | * linkage. Without this, sg_kfree() may get |
936 | * confused. |
934 | * confused. |
937 | */ |
935 | */ |
938 | if (prv) |
936 | if (prv) |
939 | table->nents = ++table->orig_nents; |
937 | table->nents = ++table->orig_nents; |
940 | 938 | ||
941 | goto err; |
939 | goto err; |
942 | } |
940 | } |
943 | 941 | ||
944 | sg_init_table(sg, alloc_size); |
942 | sg_init_table(sg, alloc_size); |
945 | table->nents = table->orig_nents += sg_size; |
943 | table->nents = table->orig_nents += sg_size; |
946 | 944 | ||
947 | /* |
945 | /* |
948 | * If this is the first mapping, assign the sg table header. |
946 | * If this is the first mapping, assign the sg table header. |
949 | * If this is not the first mapping, chain previous part. |
947 | * If this is not the first mapping, chain previous part. |
950 | */ |
948 | */ |
951 | if (prv) |
949 | if (prv) |
952 | sg_chain(prv, max_ents, sg); |
950 | sg_chain(prv, max_ents, sg); |
953 | else |
951 | else |
954 | table->sgl = sg; |
952 | table->sgl = sg; |
955 | 953 | ||
956 | /* |
954 | /* |
957 | * If no more entries after this one, mark the end |
955 | * If no more entries after this one, mark the end |
958 | */ |
956 | */ |
959 | if (!left) |
957 | if (!left) |
960 | sg_mark_end(&sg[sg_size - 1]); |
958 | sg_mark_end(&sg[sg_size - 1]); |
961 | 959 | ||
962 | prv = sg; |
960 | prv = sg; |
963 | } while (left); |
961 | } while (left); |
964 | 962 | ||
965 | return 0; |
963 | return 0; |
966 | 964 | ||
967 | err: |
965 | err: |
968 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
966 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
969 | 967 | ||
970 | return -ENOMEM; |
968 | return -ENOMEM; |
971 | } |
969 | } |
972 | 970 | ||
973 | 971 | ||
974 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
972 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
975 | { |
973 | { |
976 | memset(sgl, 0, sizeof(*sgl) * nents); |
974 | memset(sgl, 0, sizeof(*sgl) * nents); |
977 | #ifdef CONFIG_DEBUG_SG |
975 | #ifdef CONFIG_DEBUG_SG |
978 | { |
976 | { |
979 | unsigned int i; |
977 | unsigned int i; |
980 | for (i = 0; i < nents; i++) |
978 | for (i = 0; i < nents; i++) |
981 | sgl[i].sg_magic = SG_MAGIC; |
979 | sgl[i].sg_magic = SG_MAGIC; |
982 | } |
980 | } |
983 | #endif |
981 | #endif |
984 | sg_mark_end(&sgl[nents - 1]); |
982 | sg_mark_end(&sgl[nents - 1]); |
985 | } |
983 | } |
986 | 984 | ||
987 | 985 | ||
988 | void __sg_page_iter_start(struct sg_page_iter *piter, |
986 | void __sg_page_iter_start(struct sg_page_iter *piter, |
989 | struct scatterlist *sglist, unsigned int nents, |
987 | struct scatterlist *sglist, unsigned int nents, |
990 | unsigned long pgoffset) |
988 | unsigned long pgoffset) |
991 | { |
989 | { |
992 | piter->__pg_advance = 0; |
990 | piter->__pg_advance = 0; |
993 | piter->__nents = nents; |
991 | piter->__nents = nents; |
994 | 992 | ||
995 | piter->sg = sglist; |
993 | piter->sg = sglist; |
996 | piter->sg_pgoffset = pgoffset; |
994 | piter->sg_pgoffset = pgoffset; |
997 | } |
995 | } |
998 | 996 | ||
999 | static int sg_page_count(struct scatterlist *sg) |
997 | static int sg_page_count(struct scatterlist *sg) |
1000 | { |
998 | { |
1001 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
999 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
1002 | } |
1000 | } |
1003 | 1001 | ||
1004 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
1002 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
1005 | { |
1003 | { |
1006 | if (!piter->__nents || !piter->sg) |
1004 | if (!piter->__nents || !piter->sg) |
1007 | return false; |
1005 | return false; |
1008 | 1006 | ||
1009 | piter->sg_pgoffset += piter->__pg_advance; |
1007 | piter->sg_pgoffset += piter->__pg_advance; |
1010 | piter->__pg_advance = 1; |
1008 | piter->__pg_advance = 1; |
1011 | 1009 | ||
1012 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
1010 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
1013 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
1011 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
1014 | piter->sg = sg_next(piter->sg); |
1012 | piter->sg = sg_next(piter->sg); |
1015 | if (!--piter->__nents || !piter->sg) |
1013 | if (!--piter->__nents || !piter->sg) |
1016 | return false; |
1014 | return false; |
1017 | } |
1015 | } |
1018 | 1016 | ||
1019 | return true; |
1017 | return true; |
1020 | } |
1018 | } |
1021 | EXPORT_SYMBOL(__sg_page_iter_next);>=>><>20))))><20))))>20)><20)>>><>><>=>>>>>>>>>=><=>>><>><>><>><>><>><> |
1019 | EXPORT_SYMBOL(__sg_page_iter_next);>=>><>20))))><20))))>20)><20)>>><>><>=>>>>>>>>>=><=>>><>><>><>><>><>><> |