Subversion Repositories Kolibri OS

Rev

Rev 4126 | Rev 4539 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 4126 Rev 4280
1 
/*
 1 
/*
2 
 * Copyright © 2010 Daniel Vetter
 2 
 * Copyright © 2010 Daniel Vetter
3 
 *
 3 
 *
4 
 * Permission is hereby granted, free of charge, to any person obtaining a
 4 
 * Permission is hereby granted, free of charge, to any person obtaining a
5 
 * copy of this software and associated documentation files (the "Software"),
 5 
 * copy of this software and associated documentation files (the "Software"),
6 
 * to deal in the Software without restriction, including without limitation
 6 
 * to deal in the Software without restriction, including without limitation
7 
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 7 
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 
 * and/or sell copies of the Software, and to permit persons to whom the
 8 
 * and/or sell copies of the Software, and to permit persons to whom the
9 
 * Software is furnished to do so, subject to the following conditions:
 9 
 * Software is furnished to do so, subject to the following conditions:
10 
 *
 10 
 *
11 
 * The above copyright notice and this permission notice (including the next
 11 
 * The above copyright notice and this permission notice (including the next
12 
 * paragraph) shall be included in all copies or substantial portions of the
 12 
 * paragraph) shall be included in all copies or substantial portions of the
13 
 * Software.
 13 
 * Software.
14 
 *
 14 
 *
15 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 18 
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 19 
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 20 
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 
 * IN THE SOFTWARE.
 21 
 * IN THE SOFTWARE.
22 
 *
 22 
 *
23 
 */
 23 
 */
24 
 
 24 
 
25 
 
 25 
 
26 
#define AGP_NORMAL_MEMORY 0
 26 
#define AGP_NORMAL_MEMORY 0
27 
 
 27 
 
28 
#define AGP_USER_TYPES (1 << 16)
 28 
#define AGP_USER_TYPES (1 << 16)
29 
#define AGP_USER_MEMORY (AGP_USER_TYPES)
 29 
#define AGP_USER_MEMORY (AGP_USER_TYPES)
30 
#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
 30 
#define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1)
31 
 
 31 
 
32 
#include
32 
#include
33 
#include
33 
#include
34 
#include "i915_drv.h"
 34 
#include "i915_drv.h"
35 
#include "i915_trace.h"
 35 
#include "i915_trace.h"
36 
#include "intel_drv.h"
 36 
#include "intel_drv.h"
37 
 
 37 
 
38 
#define GEN6_PPGTT_PD_ENTRIES 512
 38 
#define GEN6_PPGTT_PD_ENTRIES 512
39 
#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
 39 
#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
40 
 
 40 
 
41 
/* PPGTT stuff */
 41 
/* PPGTT stuff */
42 
#define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))
 42 
#define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))
43 
#define HSW_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0x7f0))
 43 
#define HSW_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0x7f0))
44 
 
 44 
 
45 
#define GEN6_PDE_VALID			(1 << 0)
 45 
#define GEN6_PDE_VALID			(1 << 0)
46 
/* gen6+ has bit 11-4 for physical addr bit 39-32 */
 46 
/* gen6+ has bit 11-4 for physical addr bit 39-32 */
47 
#define GEN6_PDE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
 47 
#define GEN6_PDE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
48 
 
 48 
 
49 
#define GEN6_PTE_VALID			(1 << 0)
 49 
#define GEN6_PTE_VALID			(1 << 0)
50 
#define GEN6_PTE_UNCACHED		(1 << 1)
 50 
#define GEN6_PTE_UNCACHED		(1 << 1)
51 
#define HSW_PTE_UNCACHED		(0)
 51 
#define HSW_PTE_UNCACHED		(0)
52 
#define GEN6_PTE_CACHE_LLC		(2 << 1)
 52 
#define GEN6_PTE_CACHE_LLC		(2 << 1)
53 
#define GEN7_PTE_CACHE_L3_LLC		(3 << 1)
 53 
#define GEN7_PTE_CACHE_L3_LLC		(3 << 1)
54 
#define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
 54 
#define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
55 
#define HSW_PTE_ADDR_ENCODE(addr)	HSW_GTT_ADDR_ENCODE(addr)
 55 
#define HSW_PTE_ADDR_ENCODE(addr)	HSW_GTT_ADDR_ENCODE(addr)
56 
 
 56 
 
57 
/* Cacheability Control is a 4-bit value. The low three bits are stored in *
 57 
/* Cacheability Control is a 4-bit value. The low three bits are stored in *
58 
 * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
 58 
 * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
59 
 */
 59 
 */
60 
#define HSW_CACHEABILITY_CONTROL(bits)	((((bits) & 0x7) << 1) | \
 60 
#define HSW_CACHEABILITY_CONTROL(bits)	((((bits) & 0x7) << 1) | \
61 
					 (((bits) & 0x8) << (11 - 3)))
 61 
					 (((bits) & 0x8) << (11 - 3)))
62 
#define HSW_WB_LLC_AGE3			HSW_CACHEABILITY_CONTROL(0x2)
 62 
#define HSW_WB_LLC_AGE3			HSW_CACHEABILITY_CONTROL(0x2)
63 
#define HSW_WB_LLC_AGE0			HSW_CACHEABILITY_CONTROL(0x3)
 63 
#define HSW_WB_LLC_AGE0			HSW_CACHEABILITY_CONTROL(0x3)
64 
#define HSW_WB_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0xb)
 64 
#define HSW_WB_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0xb)
65 
#define HSW_WT_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0x6)
 65 
#define HSW_WT_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0x6)
66 
 
 66 
 
67 
static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
 67 
static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
68 
				     enum i915_cache_level level)
 68 
				     enum i915_cache_level level,
- 
 
 69 
				     bool valid)
69 
{
 70 
{
70 
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
 71 
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
71 
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 72 
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
72 
 
 73 
 
73 
	switch (level) {
 74 
	switch (level) {
74 
	case I915_CACHE_L3_LLC:
 75 
	case I915_CACHE_L3_LLC:
75 
	case I915_CACHE_LLC:
 76 
	case I915_CACHE_LLC:
76 
		pte |= GEN6_PTE_CACHE_LLC;
 77 
		pte |= GEN6_PTE_CACHE_LLC;
77 
		break;
 78 
		break;
78 
	case I915_CACHE_NONE:
 79 
	case I915_CACHE_NONE:
79 
		pte |= GEN6_PTE_UNCACHED;
 80 
		pte |= GEN6_PTE_UNCACHED;
80 
		break;
 81 
		break;
81 
	default:
 82 
	default:
82 
		WARN_ON(1);
 83 
		WARN_ON(1);
83 
	}
 84 
	}
84 
 
 85 
 
85 
	return pte;
 86 
	return pte;
86 
}
 87 
}
87 
 
 88 
 
88 
static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
 89 
static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
89 
				   enum i915_cache_level level)
 90 
				     enum i915_cache_level level,
- 
 
 91 
				     bool valid)
90 
{
 92 
{
91 
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
 93 
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
92 
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 94 
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
93 
 
 95 
 
94 
	switch (level) {
 96 
	switch (level) {
95 
	case I915_CACHE_L3_LLC:
 97 
	case I915_CACHE_L3_LLC:
96 
		pte |= GEN7_PTE_CACHE_L3_LLC;
 98 
		pte |= GEN7_PTE_CACHE_L3_LLC;
97 
		break;
 99 
		break;
98 
	case I915_CACHE_LLC:
 100 
	case I915_CACHE_LLC:
99 
		pte |= GEN6_PTE_CACHE_LLC;
 101 
		pte |= GEN6_PTE_CACHE_LLC;
100 
		break;
 102 
		break;
101 
	case I915_CACHE_NONE:
 103 
	case I915_CACHE_NONE:
102 
			pte |= GEN6_PTE_UNCACHED;
 104 
			pte |= GEN6_PTE_UNCACHED;
103 
		break;
 105 
		break;
104 
	default:
 106 
	default:
105 
		WARN_ON(1);
 107 
		WARN_ON(1);
106 
	}
 108 
	}
107 
 
 109 
 
108 
	return pte;
 110 
	return pte;
109 
}
 111 
}
110 
 
 112 
 
111 
#define BYT_PTE_WRITEABLE		(1 << 1)
 113 
#define BYT_PTE_WRITEABLE		(1 << 1)
112 
#define BYT_PTE_SNOOPED_BY_CPU_CACHES	(1 << 2)
 114 
#define BYT_PTE_SNOOPED_BY_CPU_CACHES	(1 << 2)
113 
 
 115 
 
114 
static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
 116 
static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
115 
				     enum i915_cache_level level)
 117 
				     enum i915_cache_level level,
- 
 
 118 
				     bool valid)
116 
{
 119 
{
117 
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
 120 
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
118 
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 121 
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
119 
 
 122 
 
120 
	/* Mark the page as writeable.  Other platforms don't have a
 123 
	/* Mark the page as writeable.  Other platforms don't have a
121 
	 * setting for read-only/writable, so this matches that behavior.
 124 
	 * setting for read-only/writable, so this matches that behavior.
122 
	 */
 125 
	 */
123 
	pte |= BYT_PTE_WRITEABLE;
 126 
	pte |= BYT_PTE_WRITEABLE;
124 
 
 127 
 
125 
	if (level != I915_CACHE_NONE)
 128 
	if (level != I915_CACHE_NONE)
126 
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
 129 
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
127 
 
 130 
 
128 
	return pte;
 131 
	return pte;
129 
}
 132 
}
130 
 
 133 
 
131 
static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
 134 
static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
132 
				     enum i915_cache_level level)
 135 
				     enum i915_cache_level level,
- 
 
 136 
				     bool valid)
133 
{
 137 
{
134 
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
 138 
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
135 
	pte |= HSW_PTE_ADDR_ENCODE(addr);
 139 
	pte |= HSW_PTE_ADDR_ENCODE(addr);
136 
 
 140 
 
137 
	if (level != I915_CACHE_NONE)
 141 
	if (level != I915_CACHE_NONE)
138 
		pte |= HSW_WB_LLC_AGE3;
 142 
		pte |= HSW_WB_LLC_AGE3;
139 
 
 143 
 
140 
	return pte;
 144 
	return pte;
141 
}
 145 
}
142 
 
 146 
 
143 
static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 147 
static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
144 
				      enum i915_cache_level level)
 148 
				      enum i915_cache_level level,
- 
 
 149 
				      bool valid)
145 
{
 150 
{
146 
	gen6_gtt_pte_t pte = GEN6_PTE_VALID;
 151 
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
147 
	pte |= HSW_PTE_ADDR_ENCODE(addr);
 152 
	pte |= HSW_PTE_ADDR_ENCODE(addr);
148 
 
 153 
 
149 
	switch (level) {
 154 
	switch (level) {
150 
	case I915_CACHE_NONE:
 155 
	case I915_CACHE_NONE:
151 
		break;
 156 
		break;
152 
	case I915_CACHE_WT:
 157 
	case I915_CACHE_WT:
153 
		pte |= HSW_WT_ELLC_LLC_AGE0;
 158 
		pte |= HSW_WT_ELLC_LLC_AGE0;
154 
		break;
 159 
		break;
155 
	default:
 160 
	default:
156 
		pte |= HSW_WB_ELLC_LLC_AGE0;
 161 
		pte |= HSW_WB_ELLC_LLC_AGE0;
157 
		break;
 162 
		break;
158 
	}
 163 
	}
159 
 
 164 
 
160 
	return pte;
 165 
	return pte;
161 
}
 166 
}
162 
 
 167 
 
163 
static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
 168 
static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
164 
{
 169 
{
165 
	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 170 
	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
166 
	gen6_gtt_pte_t __iomem *pd_addr;
 171 
	gen6_gtt_pte_t __iomem *pd_addr;
167 
	uint32_t pd_entry;
 172 
	uint32_t pd_entry;
168 
	int i;
 173 
	int i;
169 
 
 174 
 
170 
	WARN_ON(ppgtt->pd_offset & 0x3f);
 175 
	WARN_ON(ppgtt->pd_offset & 0x3f);
171 
	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
 176 
	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
172 
		ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
 177 
		ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
173 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
 178 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
174 
		dma_addr_t pt_addr;
 179 
		dma_addr_t pt_addr;
175 
 
 180 
 
176 
		pt_addr = ppgtt->pt_dma_addr[i];
 181 
		pt_addr = ppgtt->pt_dma_addr[i];
177 
		pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
 182 
		pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
178 
		pd_entry |= GEN6_PDE_VALID;
 183 
		pd_entry |= GEN6_PDE_VALID;
179 
 
 184 
 
180 
		writel(pd_entry, pd_addr + i);
 185 
		writel(pd_entry, pd_addr + i);
181 
	}
 186 
	}
182 
	readl(pd_addr);
 187 
	readl(pd_addr);
183 
}
 188 
}
184 
 
 189 
 
185 
static int gen6_ppgtt_enable(struct drm_device *dev)
 190 
static int gen6_ppgtt_enable(struct drm_device *dev)
186 
{
 191 
{
187 
	drm_i915_private_t *dev_priv = dev->dev_private;
 192 
	drm_i915_private_t *dev_priv = dev->dev_private;
188 
	uint32_t pd_offset;
 193 
	uint32_t pd_offset;
189 
	struct intel_ring_buffer *ring;
 194 
	struct intel_ring_buffer *ring;
190 
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 195 
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
191 
	int i;
 196 
	int i;
192 
 
 197 
 
193 
	BUG_ON(ppgtt->pd_offset & 0x3f);
 198 
	BUG_ON(ppgtt->pd_offset & 0x3f);
194 
 
 199 
 
195 
	gen6_write_pdes(ppgtt);
 200 
	gen6_write_pdes(ppgtt);
196 
 
 201 
 
197 
	pd_offset = ppgtt->pd_offset;
 202 
	pd_offset = ppgtt->pd_offset;
198 
	pd_offset /= 64; /* in cachelines, */
 203 
	pd_offset /= 64; /* in cachelines, */
199 
	pd_offset <<= 16;
 204 
	pd_offset <<= 16;
200 
 
 205 
 
201 
	if (INTEL_INFO(dev)->gen == 6) {
 206 
	if (INTEL_INFO(dev)->gen == 6) {
202 
		uint32_t ecochk, gab_ctl, ecobits;
 207 
		uint32_t ecochk, gab_ctl, ecobits;
203 
 
 208 
 
204 
		ecobits = I915_READ(GAC_ECO_BITS);
 209 
		ecobits = I915_READ(GAC_ECO_BITS);
205 
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
 210 
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
206 
					 ECOBITS_PPGTT_CACHE64B);
 211 
					 ECOBITS_PPGTT_CACHE64B);
207 
 
 212 
 
208 
		gab_ctl = I915_READ(GAB_CTL);
 213 
		gab_ctl = I915_READ(GAB_CTL);
209 
		I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
 214 
		I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
210 
 
 215 
 
211 
		ecochk = I915_READ(GAM_ECOCHK);
 216 
		ecochk = I915_READ(GAM_ECOCHK);
212 
		I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
 217 
		I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
213 
				       ECOCHK_PPGTT_CACHE64B);
 218 
				       ECOCHK_PPGTT_CACHE64B);
214 
		I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 219 
		I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
215 
	} else if (INTEL_INFO(dev)->gen >= 7) {
 220 
	} else if (INTEL_INFO(dev)->gen >= 7) {
216 
		uint32_t ecochk, ecobits;
 221 
		uint32_t ecochk, ecobits;
217 
 
 222 
 
218 
		ecobits = I915_READ(GAC_ECO_BITS);
 223 
		ecobits = I915_READ(GAC_ECO_BITS);
219 
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 224 
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
220 
 
 225 
 
221 
		ecochk = I915_READ(GAM_ECOCHK);
 226 
		ecochk = I915_READ(GAM_ECOCHK);
222 
		if (IS_HASWELL(dev)) {
 227 
		if (IS_HASWELL(dev)) {
223 
			ecochk |= ECOCHK_PPGTT_WB_HSW;
 228 
			ecochk |= ECOCHK_PPGTT_WB_HSW;
224 
		} else {
 229 
		} else {
225 
			ecochk |= ECOCHK_PPGTT_LLC_IVB;
 230 
			ecochk |= ECOCHK_PPGTT_LLC_IVB;
226 
			ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
 231 
			ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
227 
		}
 232 
		}
228 
		I915_WRITE(GAM_ECOCHK, ecochk);
 233 
		I915_WRITE(GAM_ECOCHK, ecochk);
229 
		/* GFX_MODE is per-ring on gen7+ */
 234 
		/* GFX_MODE is per-ring on gen7+ */
230 
	}
 235 
	}
231 
 
 236 
 
232 
	for_each_ring(ring, dev_priv, i) {
 237 
	for_each_ring(ring, dev_priv, i) {
233 
		if (INTEL_INFO(dev)->gen >= 7)
 238 
		if (INTEL_INFO(dev)->gen >= 7)
234 
			I915_WRITE(RING_MODE_GEN7(ring),
 239 
			I915_WRITE(RING_MODE_GEN7(ring),
235 
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 240 
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
236 
 
 241 
 
237 
		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 242 
		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
238 
		I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
 243 
		I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
239 
	}
 244 
	}
240 
	return 0;
 245 
	return 0;
241 
}
 246 
}
242 
 
 247 
 
243 
/* PPGTT support for Sandybdrige/Gen6 and later */
 248 
/* PPGTT support for Sandybdrige/Gen6 and later */
244 
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 249 
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
245 
				   unsigned first_entry,
 250 
				   unsigned first_entry,
246 
				   unsigned num_entries)
 251 
				   unsigned num_entries,
- 
 
 252 
				   bool use_scratch)
247 
{
 253 
{
248 
	struct i915_hw_ppgtt *ppgtt =
 254 
	struct i915_hw_ppgtt *ppgtt =
249 
		container_of(vm, struct i915_hw_ppgtt, base);
 255 
		container_of(vm, struct i915_hw_ppgtt, base);
250 
	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
 256 
	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
251 
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 257 
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
252 
	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 258 
	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
253 
	unsigned last_pte, i;
 259 
	unsigned last_pte, i;
254 
 
 260 
 
255 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
 261 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
256 
 
 262 
 
257 
    pt_vaddr = AllocKernelSpace(4096);
 263 
    pt_vaddr = AllocKernelSpace(4096);
258 
 
 264 
 
259 
    if(pt_vaddr == NULL)
 265 
    if(pt_vaddr == NULL)
260 
        return;
 266 
        return;
261 
 
 267 
 
262 
	while (num_entries) {
 268 
	while (num_entries) {
263 
            last_pte = first_pte + num_entries;
 269 
            last_pte = first_pte + num_entries;
264 
            if (last_pte > I915_PPGTT_PT_ENTRIES)
 270 
            if (last_pte > I915_PPGTT_PT_ENTRIES)
265 
                last_pte = I915_PPGTT_PT_ENTRIES;
 271 
                last_pte = I915_PPGTT_PT_ENTRIES;
266 
 
 272 
 
267 
            MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
 273 
            MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
268 
 
 274 
 
269 
            for (i = first_pte; i < last_pte; i++)
 275 
            for (i = first_pte; i < last_pte; i++)
270 
                pt_vaddr[i] = scratch_pte;
 276 
                pt_vaddr[i] = scratch_pte;
271 
 
 277 
 
272 
            num_entries -= last_pte - first_pte;
 278 
            num_entries -= last_pte - first_pte;
273 
            first_pte = 0;
 279 
            first_pte = 0;
274 
            act_pt++;
 280 
            act_pt++;
275 
    };
 281 
    };
276 
 
 282 
 
277 
    FreeKernelSpace(pt_vaddr);
 283 
    FreeKernelSpace(pt_vaddr);
278 
}
 284 
}
279 
 
 285 
 
280 
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 286 
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
281 
				      struct sg_table *pages,
 287 
				      struct sg_table *pages,
282 
				      unsigned first_entry,
 288 
				      unsigned first_entry,
283 
				      enum i915_cache_level cache_level)
 289 
				      enum i915_cache_level cache_level)
284 
{
 290 
{
285 
	struct i915_hw_ppgtt *ppgtt =
 291 
	struct i915_hw_ppgtt *ppgtt =
286 
		container_of(vm, struct i915_hw_ppgtt, base);
 292 
		container_of(vm, struct i915_hw_ppgtt, base);
287 
	gen6_gtt_pte_t *pt_vaddr;
 293 
	gen6_gtt_pte_t *pt_vaddr;
288 
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
 294 
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
289 
	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 295 
	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
290 
	struct sg_page_iter sg_iter;
 296 
	struct sg_page_iter sg_iter;
291 
	dma_addr_t page_addr;
 297 
	dma_addr_t page_addr;
292 
 
 298 
 
293 
 
 299 
 
294 
    pt_vaddr = AllocKernelSpace(4096);
 300 
    pt_vaddr = AllocKernelSpace(4096);
295 
 
 301 
 
296 
    if(pt_vaddr == NULL)
 302 
    if(pt_vaddr == NULL)
297 
        return;
 303 
        return;
298 
 
 304 
 
299 
    MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
 305 
    MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
300 
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 306 
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
301 
		dma_addr_t page_addr;
 307 
		dma_addr_t page_addr;
302 
 
 308 
 
303 
		page_addr = sg_page_iter_dma_address(&sg_iter);
 309 
		page_addr = sg_page_iter_dma_address(&sg_iter);
304 
		pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level);
 310 
		pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level, true);
305 
		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
 311 
		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
306 
			act_pt++;
 312 
			act_pt++;
307 
    		MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
 313 
    		MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
308 
			act_pte = 0;
 314 
			act_pte = 0;
309 
 
 315 
 
310 
			}
 316 
			}
311 
		}
 317 
		}
312 
    FreeKernelSpace(pt_vaddr);
 318 
    FreeKernelSpace(pt_vaddr);
313 
}
 319 
}
314 
 
 320 
 
315 
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 321 
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
316 
{
 322 
{
317 
	struct i915_hw_ppgtt *ppgtt =
 323 
	struct i915_hw_ppgtt *ppgtt =
318 
		container_of(vm, struct i915_hw_ppgtt, base);
 324 
		container_of(vm, struct i915_hw_ppgtt, base);
319 
	int i;
 325 
	int i;
320 
 
 326 
 
321 
	drm_mm_takedown(&ppgtt->base.mm);
 327 
	drm_mm_takedown(&ppgtt->base.mm);
322 
 
 328 
 
323 
	if (ppgtt->pt_dma_addr) {
 329 
	if (ppgtt->pt_dma_addr) {
324 
		for (i = 0; i < ppgtt->num_pd_entries; i++)
 330 
		for (i = 0; i < ppgtt->num_pd_entries; i++)
325 
			pci_unmap_page(ppgtt->base.dev->pdev,
 331 
			pci_unmap_page(ppgtt->base.dev->pdev,
326 
				       ppgtt->pt_dma_addr[i],
 332 
				       ppgtt->pt_dma_addr[i],
327 
				       4096, PCI_DMA_BIDIRECTIONAL);
 333 
				       4096, PCI_DMA_BIDIRECTIONAL);
328 
	}
 334 
	}
329 
 
 335 
 
330 
	kfree(ppgtt->pt_dma_addr);
 336 
	kfree(ppgtt->pt_dma_addr);
331 
	for (i = 0; i < ppgtt->num_pd_entries; i++)
 337 
	for (i = 0; i < ppgtt->num_pd_entries; i++)
332 
		__free_page(ppgtt->pt_pages[i]);
 338 
		__free_page(ppgtt->pt_pages[i]);
333 
	kfree(ppgtt->pt_pages);
 339 
	kfree(ppgtt->pt_pages);
334 
	kfree(ppgtt);
 340 
	kfree(ppgtt);
335 
}
 341 
}
336 
 
 342 
 
337 
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 343 
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
338 
{
 344 
{
339 
	struct drm_device *dev = ppgtt->base.dev;
 345 
	struct drm_device *dev = ppgtt->base.dev;
340 
	struct drm_i915_private *dev_priv = dev->dev_private;
 346 
	struct drm_i915_private *dev_priv = dev->dev_private;
341 
	unsigned first_pd_entry_in_global_pt;
 347 
	unsigned first_pd_entry_in_global_pt;
342 
	int i;
 348 
	int i;
343 
	int ret = -ENOMEM;
 349 
	int ret = -ENOMEM;
344 
 
 350 
 
345 
	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
 351 
	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
346 
	 * entries. For aliasing ppgtt support we just steal them at the end for
 352 
	 * entries. For aliasing ppgtt support we just steal them at the end for
347 
	 * now. */
 353 
	 * now. */
348 
       first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
 354 
       first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
349 
 
 355 
 
350 
	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
 356 
	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
351 
	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
 357 
	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
352 
	ppgtt->enable = gen6_ppgtt_enable;
 358 
	ppgtt->enable = gen6_ppgtt_enable;
353 
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
 359 
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
354 
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
 360 
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
355 
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
 361 
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
356 
	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
 362 
	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
357 
	ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
 363 
	ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
358 
				  GFP_KERNEL);
 364 
				  GFP_KERNEL);
359 
	if (!ppgtt->pt_pages)
 365 
	if (!ppgtt->pt_pages)
360 
		return -ENOMEM;
 366 
		return -ENOMEM;
361 
 
 367 
 
362 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
 368 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
363 
		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
 369 
		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
364 
		if (!ppgtt->pt_pages[i])
 370 
		if (!ppgtt->pt_pages[i])
365 
			goto err_pt_alloc;
 371 
			goto err_pt_alloc;
366 
	}
 372 
	}
367 
 
 373 
 
368 
	ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
 374 
	ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
369 
					     GFP_KERNEL);
 375 
					     GFP_KERNEL);
370 
    if (!ppgtt->pt_dma_addr)
 376 
    if (!ppgtt->pt_dma_addr)
371 
        goto err_pt_alloc;
 377 
        goto err_pt_alloc;
372 
 
 378 
 
373 
    for (i = 0; i < ppgtt->num_pd_entries; i++) {
 379 
    for (i = 0; i < ppgtt->num_pd_entries; i++) {
374 
        dma_addr_t pt_addr;
 380 
        dma_addr_t pt_addr;
375 
 
 381 
 
376 
		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
 382 
		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
377 
					       PCI_DMA_BIDIRECTIONAL);
 383 
					       PCI_DMA_BIDIRECTIONAL);
378 
 
 384 
 
379 
        ppgtt->pt_dma_addr[i] = pt_addr;
 385 
        ppgtt->pt_dma_addr[i] = pt_addr;
380 
    }
 386 
    }
381 
 
 387 
 
382 
	ppgtt->base.clear_range(&ppgtt->base, 0,
 388 
	ppgtt->base.clear_range(&ppgtt->base, 0,
383 
			       ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 389 
				ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true);
384 
 
 390 
 
385 
	ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
 391 
	ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
386 
 
 392 
 
387 
	return 0;
 393 
	return 0;
388 
 
 394 
 
389 
err_pd_pin:
 395 
err_pd_pin:
390 
	if (ppgtt->pt_dma_addr) {
 396 
	if (ppgtt->pt_dma_addr) {
391 
		for (i--; i >= 0; i--)
 397 
		for (i--; i >= 0; i--)
392 
			pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 398 
			pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
393 
				       4096, PCI_DMA_BIDIRECTIONAL);
 399 
				       4096, PCI_DMA_BIDIRECTIONAL);
394 
	}
 400 
	}
395 
err_pt_alloc:
 401 
err_pt_alloc:
396 
	kfree(ppgtt->pt_dma_addr);
 402 
	kfree(ppgtt->pt_dma_addr);
397 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
 403 
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
398 
		if (ppgtt->pt_pages[i])
 404 
		if (ppgtt->pt_pages[i])
399 
			__free_page(ppgtt->pt_pages[i]);
 405 
			__free_page(ppgtt->pt_pages[i]);
400 
	}
 406 
	}
401 
	kfree(ppgtt->pt_pages);
 407 
	kfree(ppgtt->pt_pages);
402 
 
 408 
 
403 
	return ret;
 409 
	return ret;
404 
}
 410 
}
405 
 
 411 
 
406 
static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
 412 
static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
407 
{
 413 
{
408 
	struct drm_i915_private *dev_priv = dev->dev_private;
 414 
	struct drm_i915_private *dev_priv = dev->dev_private;
409 
	struct i915_hw_ppgtt *ppgtt;
 415 
	struct i915_hw_ppgtt *ppgtt;
410 
	int ret;
 416 
	int ret;
411 
 
 417 
 
412 
	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
 418 
	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
413 
	if (!ppgtt)
 419 
	if (!ppgtt)
414 
		return -ENOMEM;
 420 
		return -ENOMEM;
415 
 
 421 
 
416 
	ppgtt->base.dev = dev;
 422 
	ppgtt->base.dev = dev;
417 
 
 423 
 
418 
	if (INTEL_INFO(dev)->gen < 8)
 424 
	if (INTEL_INFO(dev)->gen < 8)
419 
	ret = gen6_ppgtt_init(ppgtt);
 425 
	ret = gen6_ppgtt_init(ppgtt);
420 
	else
 426 
	else
421 
		BUG();
 427 
		BUG();
422 
 
 428 
 
423 
	if (ret)
 429 
	if (ret)
424 
        kfree(ppgtt);
 430 
        kfree(ppgtt);
425 
	else {
 431 
	else {
426 
		dev_priv->mm.aliasing_ppgtt = ppgtt;
 432 
		dev_priv->mm.aliasing_ppgtt = ppgtt;
427 
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
 433 
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
428 
			    ppgtt->base.total);
 434 
			    ppgtt->base.total);
429 
	}
 435 
	}
430 
 
 436 
 
431 
	return ret;
 437 
	return ret;
432 
}
 438 
}
433 
 
 439 
 
434 
void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 440 
void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
435 
{
 441 
{
436 
	struct drm_i915_private *dev_priv = dev->dev_private;
 442 
	struct drm_i915_private *dev_priv = dev->dev_private;
437 
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 443 
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
438 
 
 444 
 
439 
	if (!ppgtt)
 445 
	if (!ppgtt)
440 
		return;
 446 
		return;
441 
 
 447 
 
442 
	ppgtt->base.cleanup(&ppgtt->base);
 448 
	ppgtt->base.cleanup(&ppgtt->base);
443 
	dev_priv->mm.aliasing_ppgtt = NULL;
 449 
	dev_priv->mm.aliasing_ppgtt = NULL;
444 
}
 450 
}
445 
 
 451 
 
446 
void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 452 
void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
447 
			    struct drm_i915_gem_object *obj,
 453 
			    struct drm_i915_gem_object *obj,
448 
			    enum i915_cache_level cache_level)
 454 
			    enum i915_cache_level cache_level)
449 
{
 455 
{
450 
	ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
 456 
	ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
451 
				   i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
 457 
				   i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
452 
				     cache_level);
 458 
				     cache_level);
453 
}
 459 
}
454 
 
 460 
 
455 
void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 461 
void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
456 
			      struct drm_i915_gem_object *obj)
 462 
			      struct drm_i915_gem_object *obj)
457 
{
 463 
{
458 
	ppgtt->base.clear_range(&ppgtt->base,
 464 
	ppgtt->base.clear_range(&ppgtt->base,
459 
				i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
 465 
				i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
460 
			       obj->base.size >> PAGE_SHIFT);
 466 
				obj->base.size >> PAGE_SHIFT,
- 
 
 467 
				true);
461 
}
 468 
}
462 
 
 469 
 
463 
extern int intel_iommu_gfx_mapped;
 470 
extern int intel_iommu_gfx_mapped;
464 
/* Certain Gen5 chipsets require require idling the GPU before
 471 
/* Certain Gen5 chipsets require require idling the GPU before
465 
 * unmapping anything from the GTT when VT-d is enabled.
 472 
 * unmapping anything from the GTT when VT-d is enabled.
466 
 */
 473 
 */
467 
static inline bool needs_idle_maps(struct drm_device *dev)
 474 
static inline bool needs_idle_maps(struct drm_device *dev)
468 
{
 475 
{
469 
#ifdef CONFIG_INTEL_IOMMU
 476 
#ifdef CONFIG_INTEL_IOMMU
470 
	/* Query intel_iommu to see if we need the workaround. Presumably that
 477 
	/* Query intel_iommu to see if we need the workaround. Presumably that
471 
	 * was loaded first.
 478 
	 * was loaded first.
472 
	 */
 479 
	 */
473 
	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
 480 
	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
474 
		return true;
 481 
		return true;
475 
#endif
 482 
#endif
476 
	return false;
 483 
	return false;
477 
}
 484 
}
478 
 
 485 
 
479 
static bool do_idling(struct drm_i915_private *dev_priv)
 486 
static bool do_idling(struct drm_i915_private *dev_priv)
480 
{
 487 
{
481 
	bool ret = dev_priv->mm.interruptible;
 488 
	bool ret = dev_priv->mm.interruptible;
482 
 
 489 
 
483 
	if (unlikely(dev_priv->gtt.do_idle_maps)) {
 490 
	if (unlikely(dev_priv->gtt.do_idle_maps)) {
484 
		dev_priv->mm.interruptible = false;
 491 
		dev_priv->mm.interruptible = false;
485 
		if (i915_gpu_idle(dev_priv->dev)) {
 492 
		if (i915_gpu_idle(dev_priv->dev)) {
486 
			DRM_ERROR("Couldn't idle GPU\n");
 493 
			DRM_ERROR("Couldn't idle GPU\n");
487 
			/* Wait a bit, in hopes it avoids the hang */
 494 
			/* Wait a bit, in hopes it avoids the hang */
488 
			udelay(10);
 495 
			udelay(10);
489 
		}
 496 
		}
490 
	}
 497 
	}
491 
 
 498 
 
492 
	return ret;
 499 
	return ret;
493 
}
 500 
}
494 
 
 501 
 
495 
static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 502 
static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
496 
{
 503 
{
497 
	if (unlikely(dev_priv->gtt.do_idle_maps))
 504 
	if (unlikely(dev_priv->gtt.do_idle_maps))
498 
		dev_priv->mm.interruptible = interruptible;
 505 
		dev_priv->mm.interruptible = interruptible;
499 
}
 506 
}
- 
 
 507 
 
- 
 
 508 
void i915_check_and_clear_faults(struct drm_device *dev)
- 
 
 509 
{
- 
 
 510 
	struct drm_i915_private *dev_priv = dev->dev_private;
- 
 
 511 
	struct intel_ring_buffer *ring;
- 
 
 512 
	int i;
- 
 
 513 
 
- 
 
 514 
	if (INTEL_INFO(dev)->gen < 6)
- 
 
 515 
		return;
- 
 
 516 
 
- 
 
 517 
	for_each_ring(ring, dev_priv, i) {
- 
 
 518 
		u32 fault_reg;
- 
 
 519 
		fault_reg = I915_READ(RING_FAULT_REG(ring));
- 
 
 520 
		if (fault_reg & RING_FAULT_VALID) {
- 
 
 521 
			DRM_DEBUG_DRIVER("Unexpected fault\n"
- 
 
 522 
					 "\tAddr: 0x%08lx\\n"
- 
 
 523 
					 "\tAddress space: %s\n"
- 
 
 524 
					 "\tSource ID: %d\n"
- 
 
 525 
					 "\tType: %d\n",
- 
 
 526 
					 fault_reg & PAGE_MASK,
- 
 
 527 
					 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
- 
 
 528 
					 RING_FAULT_SRCID(fault_reg),
- 
 
 529 
					 RING_FAULT_FAULT_TYPE(fault_reg));
- 
 
 530 
			I915_WRITE(RING_FAULT_REG(ring),
- 
 
 531 
				   fault_reg & ~RING_FAULT_VALID);
- 
 
 532 
		}
- 
 
 533 
	}
- 
 
 534 
	POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
- 
 
 535 
}
- 
 
 536 
 
- 
 
 537 
void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
- 
 
 538 
{
- 
 
 539 
	struct drm_i915_private *dev_priv = dev->dev_private;
- 
 
 540 
 
- 
 
 541 
	/* Don't bother messing with faults pre GEN6 as we have little
- 
 
 542 
	 * documentation supporting that it's a good idea.
- 
 
 543 
	 */
- 
 
 544 
	if (INTEL_INFO(dev)->gen < 6)
- 
 
 545 
		return;
- 
 
 546 
 
- 
 
 547 
	i915_check_and_clear_faults(dev);
- 
 
 548 
 
- 
 
 549 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
- 
 
 550 
				       dev_priv->gtt.base.start / PAGE_SIZE,
- 
 
 551 
				       dev_priv->gtt.base.total / PAGE_SIZE,
- 
 
 552 
				       false);
- 
 
 553 
}
500 
 
 554 
 
501 
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 555 
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
502 
{
 556 
{
503 
	struct drm_i915_private *dev_priv = dev->dev_private;
 557 
	struct drm_i915_private *dev_priv = dev->dev_private;
504 
	struct drm_i915_gem_object *obj;
 558 
	struct drm_i915_gem_object *obj;
- 
 
 559 
 
- 
 
 560 
	i915_check_and_clear_faults(dev);
505 
 
 561 
 
506 
	/* First fill our portion of the GTT with scratch pages */
 562 
	/* First fill our portion of the GTT with scratch pages */
507 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
 563 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
508 
				       dev_priv->gtt.base.start / PAGE_SIZE,
 564 
				       dev_priv->gtt.base.start / PAGE_SIZE,
509 
				       dev_priv->gtt.base.total / PAGE_SIZE);
 565 
				       dev_priv->gtt.base.total / PAGE_SIZE,
- 
 
 566 
				       true);
510 
 
 567 
 
511 
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
 568 
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
512 
		i915_gem_clflush_object(obj, obj->pin_display);
 569 
		i915_gem_clflush_object(obj, obj->pin_display);
513 
		i915_gem_gtt_bind_object(obj, obj->cache_level);
 570 
		i915_gem_gtt_bind_object(obj, obj->cache_level);
514 
	}
 571 
	}
515 
 
 572 
 
516 
	i915_gem_chipset_flush(dev);
 573 
	i915_gem_chipset_flush(dev);
517 
}
 574 
}
518 
 
 575 
 
519 
int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 576 
int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
520 
{
 577 
{
521 
	if (obj->has_dma_mapping)
 578 
	if (obj->has_dma_mapping)
522 
		return 0;
 579 
		return 0;
523 
 
 580 
 
524 
	if (!dma_map_sg(&obj->base.dev->pdev->dev,
 581 
	if (!dma_map_sg(&obj->base.dev->pdev->dev,
525 
			obj->pages->sgl, obj->pages->nents,
 582 
			obj->pages->sgl, obj->pages->nents,
526 
			PCI_DMA_BIDIRECTIONAL))
 583 
			PCI_DMA_BIDIRECTIONAL))
527 
		return -ENOSPC;
 584 
		return -ENOSPC;
528 
 
 585 
 
529 
	return 0;
 586 
	return 0;
530 
}
 587 
}
531 
 
 588 
 
532 
/*
 589 
/*
533 
 * Binds an object into the global gtt with the specified cache level. The object
 590 
 * Binds an object into the global gtt with the specified cache level. The object
534 
 * will be accessible to the GPU via commands whose operands reference offsets
 591 
 * will be accessible to the GPU via commands whose operands reference offsets
535 
 * within the global GTT as well as accessible by the GPU through the GMADR
 592 
 * within the global GTT as well as accessible by the GPU through the GMADR
536 
 * mapped BAR (dev_priv->mm.gtt->gtt).
 593 
 * mapped BAR (dev_priv->mm.gtt->gtt).
537 
 */
 594 
 */
538 
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 595 
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
539 
				     struct sg_table *st,
 596 
				     struct sg_table *st,
540 
				     unsigned int first_entry,
 597 
				     unsigned int first_entry,
541 
				  enum i915_cache_level level)
 598 
				  enum i915_cache_level level)
542 
{
 599 
{
543 
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
 600 
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
544 
	gen6_gtt_pte_t __iomem *gtt_entries =
 601 
	gen6_gtt_pte_t __iomem *gtt_entries =
545 
		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
 602 
		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
546 
	int i = 0;
 603 
	int i = 0;
547 
	struct sg_page_iter sg_iter;
 604 
	struct sg_page_iter sg_iter;
548 
	dma_addr_t addr;
 605 
	dma_addr_t addr;
549 
 
 606 
 
550 
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
 607 
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
551 
		addr = sg_page_iter_dma_address(&sg_iter);
 608 
		addr = sg_page_iter_dma_address(&sg_iter);
552 
		iowrite32(vm->pte_encode(addr, level), >t_entries[i]);
 609 
		iowrite32(vm->pte_encode(addr, level, true), >t_entries[i]);
553 
			i++;
 610 
			i++;
554 
		}
 611 
		}
555 
 
 612 
 
556 
	/* XXX: This serves as a posting read to make sure that the PTE has
 613 
	/* XXX: This serves as a posting read to make sure that the PTE has
557 
	 * actually been updated. There is some concern that even though
 614 
	 * actually been updated. There is some concern that even though
558 
	 * registers and PTEs are within the same BAR that they are potentially
 615 
	 * registers and PTEs are within the same BAR that they are potentially
559 
	 * of NUMA access patterns. Therefore, even with the way we assume
 616 
	 * of NUMA access patterns. Therefore, even with the way we assume
560 
	 * hardware should work, we must keep this posting read for paranoia.
 617 
	 * hardware should work, we must keep this posting read for paranoia.
561 
	 */
 618 
	 */
562 
	if (i != 0)
 619 
	if (i != 0)
563 
		WARN_ON(readl(>t_entries[i-1]) !=
 620 
		WARN_ON(readl(>t_entries[i-1]) !=
564 
			vm->pte_encode(addr, level));
 621 
			vm->pte_encode(addr, level, true));
565 
 
 622 
 
566 
	/* This next bit makes the above posting read even more important. We
 623 
	/* This next bit makes the above posting read even more important. We
567 
	 * want to flush the TLBs only after we're certain all the PTE updates
 624 
	 * want to flush the TLBs only after we're certain all the PTE updates
568 
	 * have finished.
 625 
	 * have finished.
569 
	 */
 626 
	 */
570 
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 627 
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
571 
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
 628 
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
572 
}
 629 
}
573 
 
 630 
 
574 
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 631 
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
575 
				  unsigned int first_entry,
 632 
				  unsigned int first_entry,
576 
				  unsigned int num_entries)
 633 
				  unsigned int num_entries,
- 
 
 634 
				  bool use_scratch)
577 
{
 635 
{
578 
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
 636 
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
579 
	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
 637 
	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
580 
		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
 638 
		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
581 
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
 639 
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
582 
	int i;
 640 
	int i;
583 
 
 641 
 
584 
	if (WARN(num_entries > max_entries,
 642 
	if (WARN(num_entries > max_entries,
585 
		 "First entry = %d; Num entries = %d (max=%d)\n",
 643 
		 "First entry = %d; Num entries = %d (max=%d)\n",
586 
		 first_entry, num_entries, max_entries))
 644 
		 first_entry, num_entries, max_entries))
587 
        num_entries = max_entries;
 645 
        num_entries = max_entries;
588 
 
 646 
 
- 
 
 647 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch);
589 
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC);
 648 
 
590 
	for (i = 0; i < num_entries; i++)
 649 
	for (i = 0; i < num_entries; i++)
591 
		iowrite32(scratch_pte, >t_base[i]);
 650 
		iowrite32(scratch_pte, >t_base[i]);
592 
	readl(gtt_base);
 651 
	readl(gtt_base);
593 
}
 652 
}
594 
 
 653 
 
595 
 
 654 
 
596 
static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 655 
static void i915_ggtt_insert_entries(struct i915_address_space *vm,
597 
				     struct sg_table *st,
 656 
				     struct sg_table *st,
598 
				     unsigned int pg_start,
 657 
				     unsigned int pg_start,
599 
				     enum i915_cache_level cache_level)
 658 
				     enum i915_cache_level cache_level)
600 
{
 659 
{
601 
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 660 
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
602 
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 661 
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
603 
 
 662 
 
604 
	intel_gtt_insert_sg_entries(st, pg_start, flags);
 663 
	intel_gtt_insert_sg_entries(st, pg_start, flags);
605 
 
 664 
 
606 
}
 665 
}
607 
 
 666 
 
608 
static void i915_ggtt_clear_range(struct i915_address_space *vm,
 667 
static void i915_ggtt_clear_range(struct i915_address_space *vm,
609 
				  unsigned int first_entry,
 668 
				  unsigned int first_entry,
610 
				  unsigned int num_entries)
 669 
				  unsigned int num_entries,
- 
 
 670 
				  bool unused)
611 
{
 671 
{
612 
	intel_gtt_clear_range(first_entry, num_entries);
 672 
	intel_gtt_clear_range(first_entry, num_entries);
613 
}
 673 
}
614 
 
 674 
 
615 
 
 675 
 
616 
void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 676 
void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
617 
				enum i915_cache_level cache_level)
 677 
				enum i915_cache_level cache_level)
618 
{
 678 
{
619 
	struct drm_device *dev = obj->base.dev;
 679 
	struct drm_device *dev = obj->base.dev;
620 
	struct drm_i915_private *dev_priv = dev->dev_private;
 680 
	struct drm_i915_private *dev_priv = dev->dev_private;
621 
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
 681 
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
622 
 
 682 
 
623 
	dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
 683 
	dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
624 
					  entry,
 684 
					  entry,
625 
					 cache_level);
 685 
					 cache_level);
626 
 
 686 
 
627 
	obj->has_global_gtt_mapping = 1;
 687 
	obj->has_global_gtt_mapping = 1;
628 
}
 688 
}
629 
 
 689 
 
630 
void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 690 
void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
631 
{
 691 
{
632 
	struct drm_device *dev = obj->base.dev;
 692 
	struct drm_device *dev = obj->base.dev;
633 
	struct drm_i915_private *dev_priv = dev->dev_private;
 693 
	struct drm_i915_private *dev_priv = dev->dev_private;
634 
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
 694 
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
635 
 
 695 
 
636 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
 696 
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
637 
				       entry,
 697 
				       entry,
638 
			      obj->base.size >> PAGE_SHIFT);
 698 
				       obj->base.size >> PAGE_SHIFT,
- 
 
 699 
				       true);
639 
 
 700 
 
640 
	obj->has_global_gtt_mapping = 0;
 701 
	obj->has_global_gtt_mapping = 0;
641 
}
 702 
}
642 
 
 703 
 
643 
void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 704 
void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
644 
{
 705 
{
645 
	struct drm_device *dev = obj->base.dev;
 706 
	struct drm_device *dev = obj->base.dev;
646 
	struct drm_i915_private *dev_priv = dev->dev_private;
 707 
	struct drm_i915_private *dev_priv = dev->dev_private;
647 
	bool interruptible;
 708 
	bool interruptible;
648 
 
 709 
 
649 
	interruptible = do_idling(dev_priv);
 710 
	interruptible = do_idling(dev_priv);
650 
 
 711 
 
651 
	if (!obj->has_dma_mapping)
 712 
	if (!obj->has_dma_mapping)
652 
		dma_unmap_sg(&dev->pdev->dev,
 713 
		dma_unmap_sg(&dev->pdev->dev,
653 
			     obj->pages->sgl, obj->pages->nents,
 714 
			     obj->pages->sgl, obj->pages->nents,
654 
			     PCI_DMA_BIDIRECTIONAL);
 715 
			     PCI_DMA_BIDIRECTIONAL);
655 
 
 716 
 
656 
	undo_idling(dev_priv, interruptible);
 717 
	undo_idling(dev_priv, interruptible);
657 
}
 718 
}
658 
 
 719 
 
659 
static void i915_gtt_color_adjust(struct drm_mm_node *node,
 720 
static void i915_gtt_color_adjust(struct drm_mm_node *node,
660 
				  unsigned long color,
 721 
				  unsigned long color,
661 
				  unsigned long *start,
 722 
				  unsigned long *start,
662 
				  unsigned long *end)
 723 
				  unsigned long *end)
663 
{
 724 
{
664 
	if (node->color != color)
 725 
	if (node->color != color)
665 
		*start += 4096;
 726 
		*start += 4096;
666 
 
 727 
 
667 
	if (!list_empty(&node->node_list)) {
 728 
	if (!list_empty(&node->node_list)) {
668 
		node = list_entry(node->node_list.next,
 729 
		node = list_entry(node->node_list.next,
669 
				  struct drm_mm_node,
 730 
				  struct drm_mm_node,
670 
				  node_list);
 731 
				  node_list);
671 
		if (node->allocated && node->color != color)
 732 
		if (node->allocated && node->color != color)
672 
			*end -= 4096;
 733 
			*end -= 4096;
673 
	}
 734 
	}
674 
}
 735 
}
675 
void i915_gem_setup_global_gtt(struct drm_device *dev,
 736 
void i915_gem_setup_global_gtt(struct drm_device *dev,
676 
			      unsigned long start,
 737 
			      unsigned long start,
677 
			      unsigned long mappable_end,
 738 
			      unsigned long mappable_end,
678 
			      unsigned long end)
 739 
			      unsigned long end)
679 
{
 740 
{
680 
	/* Let GEM Manage all of the aperture.
 741 
	/* Let GEM Manage all of the aperture.
681 
	 *
 742 
	 *
682 
	 * However, leave one page at the end still bound to the scratch page.
 743 
	 * However, leave one page at the end still bound to the scratch page.
683 
	 * There are a number of places where the hardware apparently prefetches
 744 
	 * There are a number of places where the hardware apparently prefetches
684 
	 * past the end of the object, and we've seen multiple hangs with the
 745 
	 * past the end of the object, and we've seen multiple hangs with the
685 
	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
 746 
	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
686 
	 * aperture.  One page should be enough to keep any prefetching inside
 747 
	 * aperture.  One page should be enough to keep any prefetching inside
687 
	 * of the aperture.
 748 
	 * of the aperture.
688 
	 */
 749 
	 */
689 
	struct drm_i915_private *dev_priv = dev->dev_private;
 750 
	struct drm_i915_private *dev_priv = dev->dev_private;
690 
	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
 751 
	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
691 
	struct drm_mm_node *entry;
 752 
	struct drm_mm_node *entry;
692 
	struct drm_i915_gem_object *obj;
 753 
	struct drm_i915_gem_object *obj;
693 
	unsigned long hole_start, hole_end;
 754 
	unsigned long hole_start, hole_end;
694 
 
 755 
 
695 
	BUG_ON(mappable_end > end);
 756 
	BUG_ON(mappable_end > end);
696 
 
 757 
 
697 
	/* Subtract the guard page ... */
 758 
	/* Subtract the guard page ... */
698 
	drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
 759 
	drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
699 
	if (!HAS_LLC(dev))
 760 
	if (!HAS_LLC(dev))
700 
		dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
 761 
		dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
701 
 
 762 
 
702 
	/* Mark any preallocated objects as occupied */
 763 
	/* Mark any preallocated objects as occupied */
703 
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
 764 
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
704 
		struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
 765 
		struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
705 
		int ret;
 766 
		int ret;
706 
		DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
 767 
		DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
707 
			      i915_gem_obj_ggtt_offset(obj), obj->base.size);
 768 
			      i915_gem_obj_ggtt_offset(obj), obj->base.size);
708 
 
 769 
 
709 
		WARN_ON(i915_gem_obj_ggtt_bound(obj));
 770 
		WARN_ON(i915_gem_obj_ggtt_bound(obj));
710 
		ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
 771 
		ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
711 
		if (ret)
 772 
		if (ret)
712 
			DRM_DEBUG_KMS("Reservation failed\n");
 773 
			DRM_DEBUG_KMS("Reservation failed\n");
713 
		obj->has_global_gtt_mapping = 1;
 774 
		obj->has_global_gtt_mapping = 1;
714 
		list_add(&vma->vma_link, &obj->vma_list);
 775 
		list_add(&vma->vma_link, &obj->vma_list);
715 
	}
 776 
	}
716 
 
 777 
 
717 
	dev_priv->gtt.base.start = start;
 778 
	dev_priv->gtt.base.start = start;
718 
	dev_priv->gtt.base.total = end - start;
 779 
	dev_priv->gtt.base.total = end - start;
719 
 
 780 
 
720 
	/* Clear any non-preallocated blocks */
 781 
	/* Clear any non-preallocated blocks */
721 
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
 782 
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
722 
		const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
 783 
		const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
723 
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
 784 
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
724 
			      hole_start, hole_end);
 785 
			      hole_start, hole_end);
725 
		ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count);
 786 
		ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true);
726 
	}
 787 
	}
727 
 
 788 
 
728 
	/* And finally clear the reserved guard page */
 789 
	/* And finally clear the reserved guard page */
729 
	ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1);
 790 
	ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true);
730 
}
 791 
}
731 
 
 792 
 
732 
static bool
 793 
static bool
733 
intel_enable_ppgtt(struct drm_device *dev)
 794 
intel_enable_ppgtt(struct drm_device *dev)
734 
{
 795 
{
735 
	if (i915_enable_ppgtt >= 0)
 796 
	if (i915_enable_ppgtt >= 0)
736 
		return i915_enable_ppgtt;
 797 
		return i915_enable_ppgtt;
737 
 
 798 
 
738 
#ifdef CONFIG_INTEL_IOMMU
 799 
#ifdef CONFIG_INTEL_IOMMU
739 
	/* Disable ppgtt on SNB if VT-d is on. */
 800 
	/* Disable ppgtt on SNB if VT-d is on. */
740 
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
 801 
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
741 
		return false;
 802 
		return false;
742 
#endif
 803 
#endif
743 
 
 804 
 
744 
	return true;
 805 
	return true;
745 
}
 806 
}
746 
 
 807 
 
747 
void i915_gem_init_global_gtt(struct drm_device *dev)
 808 
void i915_gem_init_global_gtt(struct drm_device *dev)
748 
{
 809 
{
749 
	struct drm_i915_private *dev_priv = dev->dev_private;
 810 
	struct drm_i915_private *dev_priv = dev->dev_private;
750 
	unsigned long gtt_size, mappable_size;
 811 
	unsigned long gtt_size, mappable_size;
751 
 
 812 
 
752 
	gtt_size = dev_priv->gtt.base.total;
 813 
	gtt_size = dev_priv->gtt.base.total;
753 
	mappable_size = dev_priv->gtt.mappable_end;
 814 
	mappable_size = dev_priv->gtt.mappable_end;
754 
 
 - 
 
755 
#if 0
 815 
 
756 
	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
 816 
	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
757 
		int ret;
 817 
		int ret;
758 
 
 818 
 
759 
		if (INTEL_INFO(dev)->gen <= 7) {
 819 
		if (INTEL_INFO(dev)->gen <= 7) {
760 
		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
 820 
		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
761 
		 * aperture accordingly when using aliasing ppgtt. */
 821 
		 * aperture accordingly when using aliasing ppgtt. */
762 
			gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
 822 
			gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
763 
		}
 823 
		}
764 
 
 824 
 
765 
        i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size-LFB_SIZE);
 825 
		i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
766 
 
 826 
 
767 
		ret = i915_gem_init_aliasing_ppgtt(dev);
 827 
		ret = i915_gem_init_aliasing_ppgtt(dev);
768 
		if (!ret)
 828 
		if (!ret)
769 
			return;
 829 
			return;
770 
 
 830 
 
771 
		DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
 831 
		DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
772 
		drm_mm_takedown(&dev_priv->gtt.base.mm);
 832 
		drm_mm_takedown(&dev_priv->gtt.base.mm);
773 
		gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
 833 
		gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
774 
}
 834 
}
775 
#endif
 - 
 
776 
 
 - 
 
777 
    i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size-LFB_SIZE);
 835 
	i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
778 
}
 836 
}
779 
 
 837 
 
780 
static int setup_scratch_page(struct drm_device *dev)
 838 
static int setup_scratch_page(struct drm_device *dev)
781 
{
 839 
{
782 
	struct drm_i915_private *dev_priv = dev->dev_private;
 840 
	struct drm_i915_private *dev_priv = dev->dev_private;
783 
	struct page *page;
 841 
	struct page *page;
784 
	dma_addr_t dma_addr;
 842 
	dma_addr_t dma_addr;
785 
 
 843 
 
786 
	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 844 
	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
787 
	if (page == NULL)
 845 
	if (page == NULL)
788 
		return -ENOMEM;
 846 
		return -ENOMEM;
789 
    get_page(page);
 847 
    get_page(page);
790 
	set_pages_uc(page, 1);
 848 
	set_pages_uc(page, 1);
791 
 
 849 
 
792 
#ifdef CONFIG_INTEL_IOMMU
 850 
#ifdef CONFIG_INTEL_IOMMU
793 
	dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
 851 
	dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
794 
				PCI_DMA_BIDIRECTIONAL);
 852 
				PCI_DMA_BIDIRECTIONAL);
795 
	if (pci_dma_mapping_error(dev->pdev, dma_addr))
 853 
	if (pci_dma_mapping_error(dev->pdev, dma_addr))
796 
		return -EINVAL;
 854 
		return -EINVAL;
797 
#else
 855 
#else
798 
	dma_addr = page_to_phys(page);
 856 
	dma_addr = page_to_phys(page);
799 
#endif
 857 
#endif
800 
	dev_priv->gtt.base.scratch.page = page;
 858 
	dev_priv->gtt.base.scratch.page = page;
801 
	dev_priv->gtt.base.scratch.addr = dma_addr;
 859 
	dev_priv->gtt.base.scratch.addr = dma_addr;
802 
 
 860 
 
803 
	return 0;
 861 
	return 0;
804 
}
 862 
}
805 
 
 863 
 
806 
static void teardown_scratch_page(struct drm_device *dev)
 864 
static void teardown_scratch_page(struct drm_device *dev)
807 
{
 865 
{
808 
	struct drm_i915_private *dev_priv = dev->dev_private;
 866 
	struct drm_i915_private *dev_priv = dev->dev_private;
809 
	struct page *page = dev_priv->gtt.base.scratch.page;
 867 
	struct page *page = dev_priv->gtt.base.scratch.page;
810 
 
 868 
 
811 
	set_pages_wb(page, 1);
 869 
	set_pages_wb(page, 1);
812 
	pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
 870 
	pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
813 
		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 871 
		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
814 
	put_page(page);
 872 
	put_page(page);
815 
	__free_page(page);
 873 
	__free_page(page);
816 
}
 874 
}
817 
 
 875 
 
818 
static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
 876 
static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
819 
{
 877 
{
820 
	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
 878 
	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
821 
	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
 879 
	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
822 
	return snb_gmch_ctl << 20;
 880 
	return snb_gmch_ctl << 20;
823 
}
 881 
}
824 
 
 882 
 
825 
static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
 883 
static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
826 
{
 884 
{
827 
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
 885 
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
828 
	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
 886 
	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
829 
	return snb_gmch_ctl << 25; /* 32 MB units */
 887 
	return snb_gmch_ctl << 25; /* 32 MB units */
830 
}
 888 
}
831 
 
 889 
 
832 
static int gen6_gmch_probe(struct drm_device *dev,
 890 
static int gen6_gmch_probe(struct drm_device *dev,
833 
			   size_t *gtt_total,
 891 
			   size_t *gtt_total,
834 
			   size_t *stolen,
 892 
			   size_t *stolen,
835 
			   phys_addr_t *mappable_base,
 893 
			   phys_addr_t *mappable_base,
836 
			   unsigned long *mappable_end)
 894 
			   unsigned long *mappable_end)
837 
{
 895 
{
838 
	struct drm_i915_private *dev_priv = dev->dev_private;
 896 
	struct drm_i915_private *dev_priv = dev->dev_private;
839 
	phys_addr_t gtt_bus_addr;
 897 
	phys_addr_t gtt_bus_addr;
840 
	unsigned int gtt_size;
 898 
	unsigned int gtt_size;
841 
	u16 snb_gmch_ctl;
 899 
	u16 snb_gmch_ctl;
842 
	int ret;
 900 
	int ret;
843 
 
 901 
 
844 
	*mappable_base = pci_resource_start(dev->pdev, 2);
 902 
	*mappable_base = pci_resource_start(dev->pdev, 2);
845 
	*mappable_end = pci_resource_len(dev->pdev, 2);
 903 
	*mappable_end = pci_resource_len(dev->pdev, 2);
846 
 
 904 
 
847 
	/* 64/512MB is the current min/max we actually know of, but this is just
 905 
	/* 64/512MB is the current min/max we actually know of, but this is just
848 
	 * a coarse sanity check.
 906 
	 * a coarse sanity check.
849 
	 */
 907 
	 */
850 
	if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
 908 
	if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
851 
		DRM_ERROR("Unknown GMADR size (%lx)\n",
 909 
		DRM_ERROR("Unknown GMADR size (%lx)\n",
852 
			  dev_priv->gtt.mappable_end);
 910 
			  dev_priv->gtt.mappable_end);
853 
		return -ENXIO;
 911 
		return -ENXIO;
854 
		}
 912 
		}
855 
 
 913 
 
856 
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
 914 
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
857 
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
 915 
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
858 
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 916 
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
859 
	gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
 917 
	gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
860 
 
 918 
 
861 
	*stolen = gen6_get_stolen_size(snb_gmch_ctl);
 919 
	*stolen = gen6_get_stolen_size(snb_gmch_ctl);
862 
	*gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
 920 
	*gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
863 
 
 921 
 
864 
	/* For Modern GENs the PTEs and register space are split in the BAR */
 922 
	/* For Modern GENs the PTEs and register space are split in the BAR */
865 
	gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
 923 
	gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
866 
		(pci_resource_len(dev->pdev, 0) / 2);
 924 
		(pci_resource_len(dev->pdev, 0) / 2);
867 
 
 925 
 
868 
	dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
 926 
	dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
869 
	if (!dev_priv->gtt.gsm) {
 927 
	if (!dev_priv->gtt.gsm) {
870 
		DRM_ERROR("Failed to map the gtt page table\n");
 928 
		DRM_ERROR("Failed to map the gtt page table\n");
871 
		return -ENOMEM;
 929 
		return -ENOMEM;
872 
	}
 930 
	}
873 
 
 931 
 
874 
	ret = setup_scratch_page(dev);
 932 
	ret = setup_scratch_page(dev);
875 
	if (ret)
 933 
	if (ret)
876 
		DRM_ERROR("Scratch setup failed\n");
 934 
		DRM_ERROR("Scratch setup failed\n");
877 
 
 935 
 
878 
	dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
 936 
	dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
879 
	dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
 937 
	dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
880 
 
 938 
 
881 
	return ret;
 939 
	return ret;
882 
}
 940 
}
883 
 
 941 
 
884 
static void gen6_gmch_remove(struct i915_address_space *vm)
 942 
static void gen6_gmch_remove(struct i915_address_space *vm)
885 
{
 943 
{
886 
 
 944 
 
887 
	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
 945 
	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
888 
	iounmap(gtt->gsm);
 946 
	iounmap(gtt->gsm);
889 
	teardown_scratch_page(vm->dev);
 947 
	teardown_scratch_page(vm->dev);
890 
}
 948 
}
891 
 
 949 
 
892 
static int i915_gmch_probe(struct drm_device *dev,
 950 
static int i915_gmch_probe(struct drm_device *dev,
893 
			   size_t *gtt_total,
 951 
			   size_t *gtt_total,
894 
			   size_t *stolen,
 952 
			   size_t *stolen,
895 
			   phys_addr_t *mappable_base,
 953 
			   phys_addr_t *mappable_base,
896 
			   unsigned long *mappable_end)
 954 
			   unsigned long *mappable_end)
897 
{
 955 
{
898 
	struct drm_i915_private *dev_priv = dev->dev_private;
 956 
	struct drm_i915_private *dev_priv = dev->dev_private;
899 
	int ret;
 957 
	int ret;
900 
 
 958 
 
901 
	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
 959 
	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
902 
	if (!ret) {
 960 
	if (!ret) {
903 
		DRM_ERROR("failed to set up gmch\n");
 961 
		DRM_ERROR("failed to set up gmch\n");
904 
		return -EIO;
 962 
		return -EIO;
905 
	}
 963 
	}
906 
 
 964 
 
907 
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
 965 
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
908 
 
 966 
 
909 
	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
 967 
	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
910 
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
 968 
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
911 
	dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
 969 
	dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
912 
 
 970 
 
913 
	return 0;
 971 
	return 0;
914 
}
 972 
}
915 
 
 973 
 
916 
static void i915_gmch_remove(struct i915_address_space *vm)
 974 
static void i915_gmch_remove(struct i915_address_space *vm)
917 
{
 975 
{
918 
//   intel_gmch_remove();
 976 
//   intel_gmch_remove();
919 
}
 977 
}
920 
 
 978 
 
921 
int i915_gem_gtt_init(struct drm_device *dev)
 979 
int i915_gem_gtt_init(struct drm_device *dev)
922 
{
 980 
{
923 
	struct drm_i915_private *dev_priv = dev->dev_private;
 981 
	struct drm_i915_private *dev_priv = dev->dev_private;
924 
	struct i915_gtt *gtt = &dev_priv->gtt;
 982 
	struct i915_gtt *gtt = &dev_priv->gtt;
925 
	int ret;
 983 
	int ret;
926 
 
 984 
 
927 
	if (INTEL_INFO(dev)->gen <= 5) {
 985 
	if (INTEL_INFO(dev)->gen <= 5) {
928 
		gtt->gtt_probe = i915_gmch_probe;
 986 
		gtt->gtt_probe = i915_gmch_probe;
929 
		gtt->base.cleanup = i915_gmch_remove;
 987 
		gtt->base.cleanup = i915_gmch_remove;
930 
	} else {
 988 
	} else {
931 
		gtt->gtt_probe = gen6_gmch_probe;
 989 
		gtt->gtt_probe = gen6_gmch_probe;
932 
		gtt->base.cleanup = gen6_gmch_remove;
 990 
		gtt->base.cleanup = gen6_gmch_remove;
933 
		if (IS_HASWELL(dev) && dev_priv->ellc_size)
 991 
		if (IS_HASWELL(dev) && dev_priv->ellc_size)
934 
			gtt->base.pte_encode = iris_pte_encode;
 992 
			gtt->base.pte_encode = iris_pte_encode;
935 
		else if (IS_HASWELL(dev))
 993 
		else if (IS_HASWELL(dev))
936 
			gtt->base.pte_encode = hsw_pte_encode;
 994 
			gtt->base.pte_encode = hsw_pte_encode;
937 
		else if (IS_VALLEYVIEW(dev))
 995 
		else if (IS_VALLEYVIEW(dev))
938 
			gtt->base.pte_encode = byt_pte_encode;
 996 
			gtt->base.pte_encode = byt_pte_encode;
939 
		else if (INTEL_INFO(dev)->gen >= 7)
 997 
		else if (INTEL_INFO(dev)->gen >= 7)
940 
			gtt->base.pte_encode = ivb_pte_encode;
 998 
			gtt->base.pte_encode = ivb_pte_encode;
941 
		else
 999 
		else
942 
			gtt->base.pte_encode = snb_pte_encode;
 1000 
			gtt->base.pte_encode = snb_pte_encode;
943 
	}
 1001 
	}
944 
 
 1002 
 
945 
	ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size,
 1003 
	ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size,
946 
			     >t->mappable_base, >t->mappable_end);
 1004 
			     >t->mappable_base, >t->mappable_end);
947 
	if (ret)
 1005 
	if (ret)
948 
		return ret;
 1006 
		return ret;
949 
 
 1007 
 
950 
	gtt->base.dev = dev;
 1008 
	gtt->base.dev = dev;
951 
 
 1009 
 
952 
	/* GMADR is the PCI mmio aperture into the global GTT. */
 1010 
	/* GMADR is the PCI mmio aperture into the global GTT. */
953 
	DRM_INFO("Memory usable by graphics device = %zdM\n",
 1011 
	DRM_INFO("Memory usable by graphics device = %zdM\n",
954 
		 gtt->base.total >> 20);
 1012 
		 gtt->base.total >> 20);
955 
	DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
 1013 
	DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
956 
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
 1014 
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
957 
 
 1015 
 
958 
	return 0;
 1016 
	return 0;
959 
}
 1017 
}
960 
 
 1018 
 
961 
struct scatterlist *sg_next(struct scatterlist *sg)
 1019 
struct scatterlist *sg_next(struct scatterlist *sg)
962 
{
 1020 
{
963 
    if (sg_is_last(sg))
 1021 
    if (sg_is_last(sg))
964 
        return NULL;
 1022 
        return NULL;
965 
 
 1023 
 
966 
    sg++;
 1024 
    sg++;
967 
    if (unlikely(sg_is_chain(sg)))
 1025 
    if (unlikely(sg_is_chain(sg)))
968 
            sg = sg_chain_ptr(sg);
 1026 
            sg = sg_chain_ptr(sg);
969 
 
 1027 
 
970 
    return sg;
 1028 
    return sg;
971 
}
 1029 
}
972 
 
 1030 
 
973 
 
 1031 
 
974 
void __sg_free_table(struct sg_table *table, unsigned int max_ents,
 1032 
void __sg_free_table(struct sg_table *table, unsigned int max_ents,
975 
                     sg_free_fn *free_fn)
 1033 
                     sg_free_fn *free_fn)
976 
{
 1034 
{
977 
    struct scatterlist *sgl, *next;
 1035 
    struct scatterlist *sgl, *next;
978 
 
 1036 
 
979 
    if (unlikely(!table->sgl))
 1037 
    if (unlikely(!table->sgl))
980 
            return;
 1038 
            return;
981 
 
 1039 
 
982 
    sgl = table->sgl;
 1040 
    sgl = table->sgl;
983 
    while (table->orig_nents) {
 1041 
    while (table->orig_nents) {
984 
        unsigned int alloc_size = table->orig_nents;
 1042 
        unsigned int alloc_size = table->orig_nents;
985 
        unsigned int sg_size;
 1043 
        unsigned int sg_size;
986 
 
 1044 
 
987 
        /*
 1045 
        /*
988 
         * If we have more than max_ents segments left,
 1046 
         * If we have more than max_ents segments left,
989 
         * then assign 'next' to the sg table after the current one.
 1047 
         * then assign 'next' to the sg table after the current one.
990 
         * sg_size is then one less than alloc size, since the last
 1048 
         * sg_size is then one less than alloc size, since the last
991 
         * element is the chain pointer.
 1049 
         * element is the chain pointer.
992 
         */
 1050 
         */
993 
        if (alloc_size > max_ents) {
 1051 
        if (alloc_size > max_ents) {
994 
                next = sg_chain_ptr(&sgl[max_ents - 1]);
 1052 
                next = sg_chain_ptr(&sgl[max_ents - 1]);
995 
                alloc_size = max_ents;
 1053 
                alloc_size = max_ents;
996 
                sg_size = alloc_size - 1;
 1054 
                sg_size = alloc_size - 1;
997 
        } else {
 1055 
        } else {
998 
                sg_size = alloc_size;
 1056 
                sg_size = alloc_size;
999 
                next = NULL;
 1057 
                next = NULL;
1000 
        }
 1058 
        }
1001 
 
 1059 
 
1002 
        table->orig_nents -= sg_size;
 1060 
        table->orig_nents -= sg_size;
1003 
        kfree(sgl);
 1061 
        kfree(sgl);
1004 
        sgl = next;
 1062 
        sgl = next;
1005 
    }
 1063 
    }
1006 
 
 1064 
 
1007 
    table->sgl = NULL;
 1065 
    table->sgl = NULL;
1008 
}
 1066 
}
1009 
 
 1067 
 
1010 
void sg_free_table(struct sg_table *table)
 1068 
void sg_free_table(struct sg_table *table)
1011 
{
 1069 
{
1012 
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
 1070 
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
1013 
}
 1071 
}
1014 
 
 1072 
 
1015 
int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
 1073 
int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
1016 
{
 1074 
{
1017 
    struct scatterlist *sg, *prv;
 1075 
    struct scatterlist *sg, *prv;
1018 
    unsigned int left;
 1076 
    unsigned int left;
1019 
    unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
 1077 
    unsigned int max_ents = SG_MAX_SINGLE_ALLOC;
1020 
 
 1078 
 
1021 
#ifndef ARCH_HAS_SG_CHAIN
 1079 
#ifndef ARCH_HAS_SG_CHAIN
1022 
    BUG_ON(nents > max_ents);
 1080 
    BUG_ON(nents > max_ents);
1023 
#endif
 1081 
#endif
1024 
 
 1082 
 
1025 
    memset(table, 0, sizeof(*table));
 1083 
    memset(table, 0, sizeof(*table));
1026 
 
 1084 
 
1027 
    left = nents;
 1085 
    left = nents;
1028 
    prv = NULL;
 1086 
    prv = NULL;
1029 
    do {
 1087 
    do {
1030 
        unsigned int sg_size, alloc_size = left;
 1088 
        unsigned int sg_size, alloc_size = left;
1031 
 
 1089 
 
1032 
        if (alloc_size > max_ents) {
 1090 
        if (alloc_size > max_ents) {
1033 
                alloc_size = max_ents;
 1091 
                alloc_size = max_ents;
1034 
                sg_size = alloc_size - 1;
 1092 
                sg_size = alloc_size - 1;
1035 
        } else
 1093 
        } else
1036 
                sg_size = alloc_size;
 1094 
                sg_size = alloc_size;
1037 
 
 1095 
 
1038 
        left -= sg_size;
 1096 
        left -= sg_size;
1039 
 
 1097 
 
1040 
        sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
 1098 
        sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask);
1041 
        if (unlikely(!sg)) {
 1099 
        if (unlikely(!sg)) {
1042 
                /*
 1100 
                /*
1043 
                 * Adjust entry count to reflect that the last
 1101 
                 * Adjust entry count to reflect that the last
1044 
                 * entry of the previous table won't be used for
 1102 
                 * entry of the previous table won't be used for
1045 
                 * linkage.  Without this, sg_kfree() may get
 1103 
                 * linkage.  Without this, sg_kfree() may get
1046 
                 * confused.
 1104 
                 * confused.
1047 
                 */
 1105 
                 */
1048 
                if (prv)
 1106 
                if (prv)
1049 
                        table->nents = ++table->orig_nents;
 1107 
                        table->nents = ++table->orig_nents;
1050 
 
 1108 
 
1051 
                goto err;
 1109 
                goto err;
1052 
        }
 1110 
        }
1053 
 
 1111 
 
1054 
        sg_init_table(sg, alloc_size);
 1112 
        sg_init_table(sg, alloc_size);
1055 
        table->nents = table->orig_nents += sg_size;
 1113 
        table->nents = table->orig_nents += sg_size;
1056 
 
 1114 
 
1057 
        /*
 1115 
        /*
1058 
         * If this is the first mapping, assign the sg table header.
 1116 
         * If this is the first mapping, assign the sg table header.
1059 
         * If this is not the first mapping, chain previous part.
 1117 
         * If this is not the first mapping, chain previous part.
1060 
         */
 1118 
         */
1061 
        if (prv)
 1119 
        if (prv)
1062 
                sg_chain(prv, max_ents, sg);
 1120 
                sg_chain(prv, max_ents, sg);
1063 
        else
 1121 
        else
1064 
                table->sgl = sg;
 1122 
                table->sgl = sg;
1065 
 
 1123 
 
1066 
        /*
 1124 
        /*
1067 
         * If no more entries after this one, mark the end
 1125 
         * If no more entries after this one, mark the end
1068 
         */
 1126 
         */
1069 
        if (!left)
 1127 
        if (!left)
1070 
                sg_mark_end(&sg[sg_size - 1]);
 1128 
                sg_mark_end(&sg[sg_size - 1]);
1071 
 
 1129 
 
1072 
        prv = sg;
 1130 
        prv = sg;
1073 
    } while (left);
 1131 
    } while (left);
1074 
 
 1132 
 
1075 
    return 0;
 1133 
    return 0;
1076 
 
 1134 
 
1077 
err:
 1135 
err:
1078 
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
 1136 
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
1079 
 
 1137 
 
1080 
    return -ENOMEM;
 1138 
    return -ENOMEM;
1081 
}
 1139 
}
1082 
 
 1140 
 
1083 
 
 1141 
 
1084 
void sg_init_table(struct scatterlist *sgl, unsigned int nents)
 1142 
void sg_init_table(struct scatterlist *sgl, unsigned int nents)
1085 
{
 1143 
{
1086 
    memset(sgl, 0, sizeof(*sgl) * nents);
 1144 
    memset(sgl, 0, sizeof(*sgl) * nents);
1087 
#ifdef CONFIG_DEBUG_SG
 1145 
#ifdef CONFIG_DEBUG_SG
1088 
    {
 1146 
    {
1089 
            unsigned int i;
 1147 
            unsigned int i;
1090 
            for (i = 0; i < nents; i++)
 1148 
            for (i = 0; i < nents; i++)
1091 
                    sgl[i].sg_magic = SG_MAGIC;
 1149 
                    sgl[i].sg_magic = SG_MAGIC;
1092 
    }
 1150 
    }
1093 
#endif
 1151 
#endif
1094 
    sg_mark_end(&sgl[nents - 1]);
 1152 
    sg_mark_end(&sgl[nents - 1]);
1095 
}
 1153 
}
1096 
 
 1154 
 
1097 
 
 1155 
 
1098 
void __sg_page_iter_start(struct sg_page_iter *piter,
 1156 
void __sg_page_iter_start(struct sg_page_iter *piter,
1099 
              struct scatterlist *sglist, unsigned int nents,
 1157 
              struct scatterlist *sglist, unsigned int nents,
1100 
              unsigned long pgoffset)
 1158 
              unsigned long pgoffset)
1101 
{
 1159 
{
1102 
    piter->__pg_advance = 0;
 1160 
    piter->__pg_advance = 0;
1103 
    piter->__nents = nents;
 1161 
    piter->__nents = nents;
1104 
 
 1162 
 
1105 
    piter->sg = sglist;
 1163 
    piter->sg = sglist;
1106 
    piter->sg_pgoffset = pgoffset;
 1164 
    piter->sg_pgoffset = pgoffset;
1107 
}
 1165 
}
1108 
 
 1166 
 
1109 
static int sg_page_count(struct scatterlist *sg)
 1167 
static int sg_page_count(struct scatterlist *sg)
1110 
{
 1168 
{
1111 
    return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
 1169 
    return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
1112 
}
 1170 
}
1113 
 
 1171 
 
1114 
bool __sg_page_iter_next(struct sg_page_iter *piter)
 1172 
bool __sg_page_iter_next(struct sg_page_iter *piter)
1115 
{
 1173 
{
1116 
    if (!piter->__nents || !piter->sg)
 1174 
    if (!piter->__nents || !piter->sg)
1117 
        return false;
 1175 
        return false;
1118 
 
 1176 
 
1119 
    piter->sg_pgoffset += piter->__pg_advance;
 1177 
    piter->sg_pgoffset += piter->__pg_advance;
1120 
    piter->__pg_advance = 1;
 1178 
    piter->__pg_advance = 1;
1121 
 
 1179 
 
1122 
    while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
 1180 
    while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
1123 
        piter->sg_pgoffset -= sg_page_count(piter->sg);
 1181 
        piter->sg_pgoffset -= sg_page_count(piter->sg);
1124 
        piter->sg = sg_next(piter->sg);
 1182 
        piter->sg = sg_next(piter->sg);
1125 
        if (!--piter->__nents || !piter->sg)
 1183 
        if (!--piter->__nents || !piter->sg)
1126 
            return false;
 1184 
            return false;
1127 
    }
 1185 
    }
1128 
 
 1186 
 
1129 
    return true;
 1187 
    return true;
1130 
}
 1188 
}
1131 
EXPORT_SYMBOL(__sg_page_iter_next);
 1189 
EXPORT_SYMBOL(__sg_page_iter_next);