Subversion Repositories Kolibri OS

Rev

Rev 4560 | Rev 5354 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 4560 Rev 5060
Line 1... Line 1...
1
/*
1
/*
2
 * Copyright © 2010 Daniel Vetter
2
 * Copyright © 2010 Daniel Vetter
-
 
3
 * Copyright © 2011-2014 Intel Corporation
3
 *
4
 *
4
 * Permission is hereby granted, free of charge, to any person obtaining a
5
 * Permission is hereby granted, free of charge, to any person obtaining a
5
 * copy of this software and associated documentation files (the "Software"),
6
 * copy of this software and associated documentation files (the "Software"),
6
 * to deal in the Software without restriction, including without limitation
7
 * to deal in the Software without restriction, including without limitation
7
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
Line 33... Line 34...
33
#include 
34
#include 
34
#include "i915_drv.h"
35
#include "i915_drv.h"
35
#include "i915_trace.h"
36
#include "i915_trace.h"
36
#include "intel_drv.h"
37
#include "intel_drv.h"
Line 37... Line 38...
37
 
38
 
38
#define GEN6_PPGTT_PD_ENTRIES 512
39
static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
-
 
40
static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
-
 
41
 
-
 
42
bool intel_enable_ppgtt(struct drm_device *dev, bool full)
39
#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
43
{
-
 
44
	if (i915.enable_ppgtt == 0)
-
 
45
		return false;
40
typedef uint64_t gen8_gtt_pte_t;
46
 
-
 
47
	if (i915.enable_ppgtt == 1 && full)
41
typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
48
		return false;
42
 
49
 
-
 
50
	return true;
-
 
51
}
43
/* PPGTT stuff */
52
 
-
 
53
static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
44
#define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))
54
{
-
 
55
	if (enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev))
45
#define HSW_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0x7f0))
56
		return 0;
46
 
57
 
47
#define GEN6_PDE_VALID			(1 << 0)
58
	if (enable_ppgtt == 1)
48
/* gen6+ has bit 11-4 for physical addr bit 39-32 */
-
 
49
#define GEN6_PDE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
59
		return 1;
50
 
-
 
51
#define GEN6_PTE_VALID			(1 << 0)
60
 
52
#define GEN6_PTE_UNCACHED		(1 << 1)
61
	if (enable_ppgtt == 2 && HAS_PPGTT(dev))
-
 
62
		return 2;
53
#define HSW_PTE_UNCACHED		(0)
63
 
54
#define GEN6_PTE_CACHE_LLC		(2 << 1)
64
#ifdef CONFIG_INTEL_IOMMU
55
#define GEN7_PTE_CACHE_L3_LLC		(3 << 1)
65
	/* Disable ppgtt on SNB if VT-d is on. */
56
#define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
66
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
-
 
67
		DRM_INFO("Disabling PPGTT because VT-d is on\n");
-
 
68
		return 0;
-
 
69
	}
57
#define HSW_PTE_ADDR_ENCODE(addr)	HSW_GTT_ADDR_ENCODE(addr)
70
#endif
58
 
71
 
59
/* Cacheability Control is a 4-bit value. The low three bits are stored in *
-
 
60
 * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
-
 
61
 */
72
	/* Early VLV doesn't have this */
62
#define HSW_CACHEABILITY_CONTROL(bits)	((((bits) & 0x7) << 1) | \
73
	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
63
					 (((bits) & 0x8) << (11 - 3)))
74
	    dev->pdev->revision < 0xb) {
64
#define HSW_WB_LLC_AGE3			HSW_CACHEABILITY_CONTROL(0x2)
75
		DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
65
#define HSW_WB_LLC_AGE0			HSW_CACHEABILITY_CONTROL(0x3)
-
 
66
#define HSW_WB_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0xb)
-
 
67
#define HSW_WB_ELLC_LLC_AGE3		HSW_CACHEABILITY_CONTROL(0x8)
-
 
68
#define HSW_WT_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0x6)
-
 
-
 
76
		return 0;
69
#define HSW_WT_ELLC_LLC_AGE3		HSW_CACHEABILITY_CONTROL(0x7)
77
	}
70
 
-
 
71
#define GEN8_PTES_PER_PAGE		(PAGE_SIZE / sizeof(gen8_gtt_pte_t))
-
 
72
#define GEN8_PDES_PER_PAGE		(PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
78
 
-
 
79
	return HAS_ALIASING_PPGTT(dev) ? 1 : 0;
-
 
80
}
73
#define GEN8_LEGACY_PDPS		4
81
 
74
 
82
 
75
#define PPAT_UNCACHED_INDEX		(_PAGE_PWT | _PAGE_PCD)
83
static void ppgtt_bind_vma(struct i915_vma *vma,
-
 
84
			   enum i915_cache_level cache_level,
76
#define PPAT_CACHED_PDE_INDEX		0 /* WB LLC */
85
			   u32 flags);
77
#define PPAT_CACHED_INDEX		_PAGE_PAT /* WB LLCeLLC */
86
static void ppgtt_unbind_vma(struct i915_vma *vma);
Line 78... Line 87...
78
#define PPAT_DISPLAY_ELLC_INDEX		_PAGE_PCD /* WT eLLC */
87
static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
79
 
88
 
80
static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
89
static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
81
					     enum i915_cache_level level,
90
					     enum i915_cache_level level,
82
					     bool valid)
91
					     bool valid)
83
{
92
{
-
 
93
	gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
84
    gen8_gtt_pte_t pte = valid ? 1 | 2 : 0;
94
	pte |= addr;
85
	pte |= addr;
95
 
86
	if (level != I915_CACHE_NONE)
-
 
87
		pte |= PPAT_CACHED_INDEX;
96
	switch (level) {
-
 
97
	case I915_CACHE_NONE:
-
 
98
		pte |= PPAT_UNCACHED_INDEX;
-
 
99
		break;
-
 
100
	case I915_CACHE_WT:
-
 
101
		pte |= PPAT_DISPLAY_ELLC_INDEX;
-
 
102
		break;
-
 
103
	default:
-
 
104
		pte |= PPAT_CACHED_INDEX;
-
 
105
		break;
88
	else
106
	}
89
		pte |= PPAT_UNCACHED_INDEX;
107
 
Line 90... Line 108...
90
	return pte;
108
	return pte;
91
}
109
}
Line 103... Line 121...
103
	return pde;
121
	return pde;
104
}
122
}
Line 105... Line 123...
105
 
123
 
106
static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
124
static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
107
				     enum i915_cache_level level,
125
				     enum i915_cache_level level,
108
				     bool valid)
126
				     bool valid, u32 unused)
109
{
127
{
110
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
128
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Line 111... Line 129...
111
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
129
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
Line 125... Line 143...
125
	return pte;
143
	return pte;
126
}
144
}
Line 127... Line 145...
127
 
145
 
128
static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
146
static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
129
				     enum i915_cache_level level,
147
				     enum i915_cache_level level,
130
				     bool valid)
148
				     bool valid, u32 unused)
131
{
149
{
132
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
150
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Line 133... Line 151...
133
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
151
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
Line 147... Line 165...
147
	}
165
	}
Line 148... Line 166...
148
 
166
 
149
	return pte;
167
	return pte;
Line 150... Line -...
150
}
-
 
151
 
-
 
152
#define BYT_PTE_WRITEABLE		(1 << 1)
-
 
153
#define BYT_PTE_SNOOPED_BY_CPU_CACHES	(1 << 2)
168
}
154
 
169
 
155
static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
170
static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
156
				     enum i915_cache_level level,
171
				     enum i915_cache_level level,
157
				     bool valid)
172
				     bool valid, u32 flags)
158
{
173
{
Line 159... Line 174...
159
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
174
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
160
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
175
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
161
 
176
 
-
 
177
	/* Mark the page as writeable.  Other platforms don't have a
162
	/* Mark the page as writeable.  Other platforms don't have a
178
	 * setting for read-only/writable, so this matches that behavior.
Line 163... Line 179...
163
	 * setting for read-only/writable, so this matches that behavior.
179
	 */
164
	 */
180
	if (!(flags & PTE_READ_ONLY))
Line 165... Line 181...
165
	pte |= BYT_PTE_WRITEABLE;
181
	pte |= BYT_PTE_WRITEABLE;
166
 
182
 
Line 167... Line 183...
167
	if (level != I915_CACHE_NONE)
183
	if (level != I915_CACHE_NONE)
168
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
184
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
169
 
185
 
170
	return pte;
186
	return pte;
171
}
187
}
172
 
188
 
Line 173... Line 189...
173
static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
189
static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
Line 183... Line 199...
183
	return pte;
199
	return pte;
184
}
200
}
Line 185... Line 201...
185
 
201
 
186
static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
202
static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
187
				      enum i915_cache_level level,
203
				      enum i915_cache_level level,
188
				      bool valid)
204
				      bool valid, u32 unused)
189
{
205
{
190
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
206
	gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Line 191... Line 207...
191
	pte |= HSW_PTE_ADDR_ENCODE(addr);
207
	pte |= HSW_PTE_ADDR_ENCODE(addr);
Line 203... Line 219...
203
 
219
 
204
	return pte;
220
	return pte;
Line 205... Line 221...
205
}
221
}
206
 
222
 
207
/* Broadwell Page Directory Pointer Descriptors */
223
/* Broadwell Page Directory Pointer Descriptors */
208
static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
224
static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
-
 
225
			   uint64_t val, bool synchronous)
209
			   uint64_t val)
226
{
Line 210... Line 227...
210
{
227
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
Line -... Line 228...
-
 
228
	int ret;
-
 
229
 
-
 
230
	BUG_ON(entry >= 4);
-
 
231
 
-
 
232
	if (synchronous) {
-
 
233
		I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);
211
	int ret;
234
		I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);
212
 
235
		return 0;
213
	BUG_ON(entry >= 4);
236
	}
Line 214... Line 237...
214
 
237
 
Line 225... Line 248...
225
	intel_ring_advance(ring);
248
	intel_ring_advance(ring);
Line 226... Line 249...
226
 
249
 
227
	return 0;
250
	return 0;
Line 228... Line 251...
228
}
251
}
-
 
252
 
-
 
253
static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
229
 
254
			  struct intel_engine_cs *ring,
230
static int gen8_ppgtt_enable(struct drm_device *dev)
-
 
231
{
-
 
232
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
233
	struct intel_ring_buffer *ring;
255
			  bool synchronous)
Line 234... Line 256...
234
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
256
{
235
	int i, j, ret;
257
	int i, ret;
Line 236... Line -...
236
 
-
 
237
	/* bit of a hack to find the actual last used pd */
-
 
238
	int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
-
 
239
 
-
 
240
	for_each_ring(ring, dev_priv, j) {
-
 
241
		I915_WRITE(RING_MODE_GEN7(ring),
258
 
242
			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
259
	/* bit of a hack to find the actual last used pd */
243
	}
-
 
244
 
260
	int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
245
	for (i = used_pd - 1; i >= 0; i--) {
261
 
246
		dma_addr_t addr = ppgtt->pd_dma_addr[i];
262
	for (i = used_pd - 1; i >= 0; i--) {
247
		for_each_ring(ring, dev_priv, j) {
-
 
248
			ret = gen8_write_pdp(ring, i, addr);
263
		dma_addr_t addr = ppgtt->pd_dma_addr[i];
249
			if (ret)
-
 
Line 250... Line -...
250
				goto err_out;
-
 
251
		}
-
 
252
	}
-
 
253
	return 0;
-
 
254
 
264
		ret = gen8_write_pdp(ring, i, addr, synchronous);
255
err_out:
265
			if (ret)
Line 256... Line 266...
256
	for_each_ring(ring, dev_priv, j)
266
			return ret;
257
		I915_WRITE(RING_MODE_GEN7(ring),
267
	}
258
			   _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
268
 
259
	return ret;
269
	return 0;
260
}
270
}
261
 
271
 
262
static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
272
static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
263
				   unsigned first_entry,
273
				   uint64_t start,
-
 
274
				   uint64_t length,
264
				   unsigned num_entries,
275
				   bool use_scratch)
265
				   bool use_scratch)
276
{
-
 
277
	struct i915_hw_ppgtt *ppgtt =
266
{
278
		container_of(vm, struct i915_hw_ppgtt, base);
Line 267... Line 279...
267
	struct i915_hw_ppgtt *ppgtt =
279
	gen8_gtt_pte_t *pt_vaddr, scratch_pte;
268
		container_of(vm, struct i915_hw_ppgtt, base);
280
	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
269
	gen8_gtt_pte_t *pt_vaddr, scratch_pte;
281
	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
Line 270... Line 282...
270
	unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
282
	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
271
	unsigned first_pte = first_entry % GEN8_PTES_PER_PAGE;
283
	unsigned num_entries = length >> PAGE_SHIFT;
Line 272... Line 284...
272
	unsigned last_pte, i;
284
	unsigned last_pte, i;
273
 
285
 
Line 274... Line 286...
274
    pt_vaddr = (gen8_gtt_pte_t*)AllocKernelSpace(4096);
286
    pt_vaddr = (gen8_gtt_pte_t*)AllocKernelSpace(4096);
275
    if(pt_vaddr == NULL)
287
    if(pt_vaddr == NULL)
276
        return;
288
        return;
Line 277... Line 289...
277
 
289
 
Line 278... Line 290...
278
    scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
290
    scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
279
                      I915_CACHE_LLC, use_scratch);
291
                      I915_CACHE_LLC, use_scratch);
-
 
292
 
-
 
293
	while (num_entries) {
Line -... Line 294...
-
 
294
		struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
280
 
295
 
-
 
296
		last_pte = pte + num_entries;
281
	while (num_entries) {
297
		if (last_pte > GEN8_PTES_PER_PAGE)
-
 
298
			last_pte = GEN8_PTES_PER_PAGE;
282
		struct page *page_table = &ppgtt->gen8_pt_pages[act_pt];
299
 
-
 
300
        MapPage(pt_vaddr,(addr_t)page_table, PG_SW);
-
 
301
 
283
 
302
		for (i = pte; i < last_pte; i++) {
284
		last_pte = first_pte + num_entries;
303
			pt_vaddr[i] = scratch_pte;
285
		if (last_pte > GEN8_PTES_PER_PAGE)
304
			num_entries--;
Line 286... Line 305...
286
			last_pte = GEN8_PTES_PER_PAGE;
305
		}
287
 
306
 
288
        MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
307
		if (!HAS_LLC(ppgtt->base.dev))
289
 
308
			drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
290
		for (i = first_pte; i < last_pte; i++)
309
 
291
			pt_vaddr[i] = scratch_pte;
310
		pte = 0;
292
 
311
		if (++pde == GEN8_PDES_PER_PAGE) {
293
		num_entries -= last_pte - first_pte;
312
			pdpe++;
-
 
313
			pde = 0;
294
		first_pte = 0;
314
		}
295
		act_pt++;
315
	}
296
	}
316
    FreeKernelSpace(pt_vaddr);
Line 297... Line 317...
297
    FreeKernelSpace(pt_vaddr);
317
}
298
}
318
 
299
 
319
static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
Line 300... Line 320...
300
static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
320
				      struct sg_table *pages,
Line 301... Line 321...
301
				      struct sg_table *pages,
321
				      uint64_t start,
-
 
322
				      enum i915_cache_level cache_level, u32 unused)
-
 
323
{
Line 302... Line 324...
302
				      unsigned first_entry,
324
	struct i915_hw_ppgtt *ppgtt =
303
				      enum i915_cache_level cache_level)
325
		container_of(vm, struct i915_hw_ppgtt, base);
304
{
326
	gen8_gtt_pte_t *pt_vaddr;
305
	struct i915_hw_ppgtt *ppgtt =
327
	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
-
 
328
	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
-
 
329
	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
-
 
330
	struct sg_page_iter sg_iter;
306
		container_of(vm, struct i915_hw_ppgtt, base);
331
 
307
	gen8_gtt_pte_t *pt_vaddr;
332
    pt_vaddr = AllocKernelSpace(4096);
-
 
333
    if(pt_vaddr == NULL)
308
	unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
334
        return;
-
 
335
 
309
	unsigned act_pte = first_entry % GEN8_PTES_PER_PAGE;
336
    MapPage(pt_vaddr,(addr_t)(ppgtt->gen8_pt_pages[pdpe][pde]), 3);
310
	struct sg_page_iter sg_iter;
337
 
311
 
338
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
312
    pt_vaddr = AllocKernelSpace(4096);
339
		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
Line 313... Line 340...
313
    if(pt_vaddr == NULL)
340
			break;
314
        return;
341
 
315
 
-
 
316
    MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
-
 
317
 
342
		pt_vaddr[pte] =
Line 318... Line 343...
318
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
343
			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
-
 
344
					cache_level, true);
-
 
345
		if (++pte == GEN8_PTES_PER_PAGE) {
-
 
346
			if (!HAS_LLC(ppgtt->base.dev))
-
 
347
				drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
-
 
348
			if (++pde == GEN8_PDES_PER_PAGE) {
-
 
349
				pdpe++;
-
 
350
				pde = 0;
-
 
351
			}
-
 
352
			pte = 0;
-
 
353
            MapPage(pt_vaddr,(addr_t)(ppgtt->gen8_pt_pages[pdpe][pde]), 3);
Line 319... Line 354...
319
 
354
		}
-
 
355
	}
-
 
356
    FreeKernelSpace(pt_vaddr);
320
		pt_vaddr[act_pte] =
357
}
-
 
358
 
-
 
359
static void gen8_free_page_tables(struct page **pt_pages)
-
 
360
{
-
 
361
	int i;
-
 
362
 
-
 
363
	if (pt_pages == NULL)
-
 
364
		return;
321
			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
365
 
-
 
366
//   for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
-
 
367
//       if (pt_pages[i])
-
 
368
//           __free_pages(pt_pages[i], 0);
-
 
369
}
-
 
370
 
322
					cache_level, true);
371
static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
-
 
372
{
-
 
373
	int i;
-
 
374
 
323
		if (++act_pte == GEN8_PTES_PER_PAGE) {
375
	for (i = 0; i < ppgtt->num_pd_pages; i++) {
Line 324... Line 376...
324
			act_pt++;
376
		gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
325
            MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
377
		kfree(ppgtt->gen8_pt_pages[i]);
326
			act_pte = 0;
378
		kfree(ppgtt->gen8_pt_dma_addr[i]);
327
		}
379
	}
328
	}
-
 
329
    FreeKernelSpace(pt_vaddr);
-
 
330
}
380
 
331
 
-
 
332
static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
381
//	__free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
333
{
382
}
334
	struct i915_hw_ppgtt *ppgtt =
-
 
335
		container_of(vm, struct i915_hw_ppgtt, base);
383
 
Line 336... Line 384...
336
	int i, j;
384
static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
-
 
385
{
-
 
386
	struct pci_dev *hwdev = ppgtt->base.dev->pdev;
337
 
387
	int i, j;
-
 
388
 
-
 
389
	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-
 
390
		/* TODO: In the future we'll support sparse mappings, so this
-
 
391
		 * will have to change. */
-
 
392
		if (!ppgtt->pd_dma_addr[i])
-
 
393
			continue;
338
	drm_mm_takedown(&vm->mm);
394
 
Line 339... Line -...
339
 
-
 
340
	for (i = 0; i < ppgtt->num_pd_pages ; i++) {
-
 
341
		if (ppgtt->pd_dma_addr[i]) {
-
 
342
			pci_unmap_page(ppgtt->base.dev->pdev,
-
 
343
				       ppgtt->pd_dma_addr[i],
-
 
344
				       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-
 
345
 
-
 
346
			for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
-
 
347
				dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
395
		pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
348
				if (addr)
396
			       PCI_DMA_BIDIRECTIONAL);
349
					pci_unmap_page(ppgtt->base.dev->pdev,
397
 
350
						       addr,
398
			for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
351
						       PAGE_SIZE,
-
 
352
						       PCI_DMA_BIDIRECTIONAL);
-
 
Line -... Line 399...
-
 
399
				dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
-
 
400
				if (addr)
-
 
401
				pci_unmap_page(hwdev, addr, PAGE_SIZE,
-
 
402
						       PCI_DMA_BIDIRECTIONAL);
-
 
403
		}
-
 
404
	}
353
 
405
}
-
 
406
 
-
 
407
static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
-
 
408
{
-
 
409
	struct i915_hw_ppgtt *ppgtt =
-
 
410
		container_of(vm, struct i915_hw_ppgtt, base);
-
 
411
 
-
 
412
	list_del(&vm->global_link);
-
 
413
	drm_mm_takedown(&vm->mm);
-
 
414
 
-
 
415
	gen8_ppgtt_unmap_pages(ppgtt);
-
 
416
	gen8_ppgtt_free(ppgtt);
354
			}
417
}
-
 
418
 
-
 
419
static struct page **__gen8_alloc_page_tables(void)
-
 
420
{
-
 
421
	struct page **pt_pages;
-
 
422
	int i;
-
 
423
 
-
 
424
	pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
-
 
425
	if (!pt_pages)
-
 
426
		return ERR_PTR(-ENOMEM);
-
 
427
 
-
 
428
	for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
-
 
429
		pt_pages[i] = alloc_page(GFP_KERNEL);
Line 355... Line -...
355
		}
-
 
356
		kfree(ppgtt->gen8_pt_dma_addr[i]);
430
		if (!pt_pages[i])
357
	}
431
			goto bail;
358
 
432
			}
-
 
433
 
359
//   __free_pages(ppgtt->gen8_pt_pages, get_order(ppgtt->num_pt_pages << PAGE_SHIFT));
434
	return pt_pages;
-
 
435
 
-
 
436
bail:
-
 
437
	gen8_free_page_tables(pt_pages);
-
 
438
	kfree(pt_pages);
-
 
439
	return ERR_PTR(-ENOMEM);
-
 
440
}
-
 
441
 
-
 
442
static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
-
 
443
					   const int max_pdp)
-
 
444
{
-
 
445
	struct page **pt_pages[GEN8_LEGACY_PDPS];
-
 
446
	int i, ret;
-
 
447
 
-
 
448
	for (i = 0; i < max_pdp; i++) {
-
 
449
		pt_pages[i] = __gen8_alloc_page_tables();
-
 
450
		if (IS_ERR(pt_pages[i])) {
-
 
451
			ret = PTR_ERR(pt_pages[i]);
-
 
452
			goto unwind_out;
-
 
453
		}
-
 
454
	}
-
 
455
 
-
 
456
	/* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
-
 
457
	 * "atomic" - for cleanup purposes.
-
 
458
	 */
-
 
459
	for (i = 0; i < max_pdp; i++)
-
 
460
		ppgtt->gen8_pt_pages[i] = pt_pages[i];
-
 
461
 
-
 
462
	return 0;
-
 
463
 
-
 
464
unwind_out:
-
 
465
	while (i--) {
360
//   __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
466
		gen8_free_page_tables(pt_pages[i]);
361
}
467
		kfree(pt_pages[i]);
Line 362... Line 468...
362
 
468
	}
-
 
469
 
-
 
470
	return ret;
-
 
471
}
-
 
472
 
-
 
473
static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
-
 
474
{
-
 
475
	int i;
-
 
476
 
-
 
477
	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-
 
478
		ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
-
 
479
						     sizeof(dma_addr_t),
-
 
480
						     GFP_KERNEL);
-
 
481
		if (!ppgtt->gen8_pt_dma_addr[i])
-
 
482
			return -ENOMEM;
-
 
483
		}
363
/**
484
 
364
 * GEN8 legacy ppgtt programming is accomplished through 4 PDP registers with a
485
	return 0;
365
 * net effect resembling a 2-level page table in normal x86 terms. Each PDP
486
}
366
 * represents 1GB of memory
487
 
Line 367... Line -...
367
 * 4 * 512 * 512 * 4096 = 4GB legacy 32b address space.
-
 
368
 *
-
 
369
 * TODO: Do something with the size parameter
-
 
370
 **/
488
static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
371
static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
-
 
372
{
-
 
373
	struct page *pt_pages;
-
 
374
	int i, j, ret = -ENOMEM;
-
 
375
	const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
-
 
376
	const int num_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
-
 
Line 377... Line 489...
377
 
489
						const int max_pdp)
-
 
490
{
-
 
491
//	ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
Line -... Line 492...
-
 
492
	if (!ppgtt->pd_pages)
378
	if (size % (1<<30))
493
		return -ENOMEM;
-
 
494
 
379
		DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
495
//   ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
380
 
-
 
381
	/* FIXME: split allocation into smaller pieces. For now we only ever do
-
 
382
	 * this once, but with full PPGTT, the multiple contiguous allocations
496
	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
383
	 * will be bad.
497
 
384
	 */
498
	return 0;
385
    ppgtt->pd_pages = AllocPages(max_pdp);
499
}
-
 
500
 
386
	if (!ppgtt->pd_pages)
501
static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
387
		return -ENOMEM;
502
			    const int max_pdp)
388
 
503
{
Line -... Line 504...
-
 
504
	int ret;
-
 
505
 
389
    pt_pages = AllocPages(num_pt_pages);
506
	ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
Line 390... Line -...
390
	if (!pt_pages) {
-
 
391
//       __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
507
	if (ret)
392
		return -ENOMEM;
-
 
Line -... Line 508...
-
 
508
		return ret;
-
 
509
 
-
 
510
	ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
393
	}
511
	if (ret) {
-
 
512
//		__free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
-
 
513
		return ret;
-
 
514
	}
-
 
515
 
-
 
516
	ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
-
 
517
 
-
 
518
	ret = gen8_ppgtt_allocate_dma(ppgtt);
394
 
519
	if (ret)
395
	ppgtt->gen8_pt_pages = pt_pages;
520
		gen8_ppgtt_free(ppgtt);
396
    ppgtt->num_pd_pages = max_pdp;
521
 
-
 
522
	return ret;
-
 
523
}
-
 
524
 
-
 
525
static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
-
 
526
					     const int pd)
-
 
527
{
-
 
528
	dma_addr_t pd_addr;
-
 
529
	int ret;
-
 
530
 
-
 
531
	pd_addr = pci_map_page(ppgtt->base.dev->pdev,
-
 
532
			       &ppgtt->pd_pages[pd], 0,
-
 
533
			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-
 
534
 
-
 
535
//   ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
-
 
536
//   if (ret)
-
 
537
//       return ret;
-
 
538
 
-
 
539
	ppgtt->pd_dma_addr[pd] = pd_addr;
-
 
540
 
-
 
541
	return 0;
-
 
542
}
-
 
543
 
-
 
544
static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
-
 
545
					const int pd,
397
    ppgtt->num_pt_pages = num_pt_pages;
546
					const int pt)
Line -... Line 547...
-
 
547
{
-
 
548
	dma_addr_t pt_addr;
-
 
549
	struct page *p;
-
 
550
	int ret;
-
 
551
 
-
 
552
	p = ppgtt->gen8_pt_pages[pd][pt];
-
 
553
	pt_addr = pci_map_page(ppgtt->base.dev->pdev,
-
 
554
			       p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-
 
555
//   ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
-
 
556
//   if (ret)
-
 
557
//       return ret;
-
 
558
 
398
	ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
559
	ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
-
 
560
 
-
 
561
	return 0;
-
 
562
}
-
 
563
 
-
 
564
/**
-
 
565
 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
-
 
566
 * with a net effect resembling a 2-level page table in normal x86 terms. Each
399
	ppgtt->enable = gen8_ppgtt_enable;
567
 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
400
	ppgtt->base.clear_range = gen8_ppgtt_clear_range;
568
 * space.
Line -... Line 569...
-
 
569
 *
-
 
570
 * FIXME: split allocation into smaller pieces. For now we only ever do this
-
 
571
 * once, but with full PPGTT, the multiple contiguous allocations will be bad.
-
 
572
 * TODO: Do something with the size parameter
401
	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
573
 */
402
	ppgtt->base.cleanup = gen8_ppgtt_cleanup;
574
static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
403
	ppgtt->base.start = 0;
575
{
-
 
576
	const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
Line 404... Line -...
404
	ppgtt->base.total = ppgtt->num_pt_pages * GEN8_PTES_PER_PAGE * PAGE_SIZE;
-
 
405
 
577
	const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
Line 406... Line 578...
406
	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
578
	int i, j, ret;
407
 
579
	gen8_ppgtt_pde_t *pd_vaddr;
408
	/*
580
 
409
	 * - Create a mapping for the page directories.
581
	if (size % (1<<30))
410
	 * - For each page directory:
582
		DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
411
	 *      allocate space for page table mappings.
583
 
412
	 *      map each page table
584
	/* 1. Do all our allocations for page directories and page tables. */
-
 
585
	ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
-
 
586
	if (ret)
413
	 */
587
		return ret;
414
	for (i = 0; i < max_pdp; i++) {
588
 
Line -... Line 589...
-
 
589
	/*
-
 
590
	 * 2. Create DMA mappings for the page directories and page tables.
415
		dma_addr_t temp;
591
	 */
416
		temp = pci_map_page(ppgtt->base.dev->pdev,
592
	for (i = 0; i < max_pdp; i++) {
-
 
593
		ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
417
				    &ppgtt->pd_pages[i], 0,
594
		if (ret)
-
 
595
			goto bail;
-
 
596
 
-
 
597
		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
Line 418... Line 598...
418
				    PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
598
			ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
419
 
599
			if (ret)
420
		ppgtt->pd_dma_addr[i] = temp;
600
				goto bail;
421
 
601
		}
422
		ppgtt->gen8_pt_dma_addr[i] = kmalloc(sizeof(dma_addr_t) * GEN8_PDES_PER_PAGE, GFP_KERNEL);
602
	}
423
		if (!ppgtt->gen8_pt_dma_addr[i])
-
 
424
			goto err_out;
603
 
Line 425... Line 604...
425
 
604
	/*
426
		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
605
	 * 3. Map all the page directory entires to point to the page tables
-
 
606
	 * we've allocated.
427
			struct page *p = &pt_pages[i * GEN8_PDES_PER_PAGE + j];
607
	 *
428
			temp = pci_map_page(ppgtt->base.dev->pdev,
608
	 * For now, the PPGTT helper functions all require that the PDEs are
Line 429... Line 609...
429
					    p, 0, PAGE_SIZE,
609
	 * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
430
					    PCI_DMA_BIDIRECTIONAL);
610
	 * will never need to touch the PDEs again.
Line 487... Line 667...
487
		writel(pd_entry, pd_addr + i);
667
		writel(pd_entry, pd_addr + i);
488
	}
668
	}
489
	readl(pd_addr);
669
	readl(pd_addr);
490
}
670
}
Line 491... Line 671...
491
 
671
 
492
static int gen6_ppgtt_enable(struct drm_device *dev)
672
static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
493
{
-
 
494
	drm_i915_private_t *dev_priv = dev->dev_private;
-
 
495
	uint32_t pd_offset;
-
 
496
	struct intel_ring_buffer *ring;
-
 
497
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-
 
498
	int i;
-
 
499
 
673
{
Line 500... Line 674...
500
	BUG_ON(ppgtt->pd_offset & 0x3f);
674
	BUG_ON(ppgtt->pd_offset & 0x3f);
-
 
675
 
Line -... Line 676...
-
 
676
	return (ppgtt->pd_offset / 64) << 16;
-
 
677
}
-
 
678
 
-
 
679
static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
501
 
680
			 struct intel_engine_cs *ring,
502
	gen6_write_pdes(ppgtt);
681
			 bool synchronous)
503
 
682
{
Line -... Line 683...
-
 
683
	struct drm_device *dev = ppgtt->base.dev;
-
 
684
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
685
	int ret;
-
 
686
 
-
 
687
	/* If we're in reset, we can assume the GPU is sufficiently idle to
-
 
688
	 * manually frob these bits. Ideally we could use the ring functions,
-
 
689
	 * except our error handling makes it quite difficult (can't use
-
 
690
	 * intel_ring_begin, ring->flush, or intel_ring_advance)
504
	pd_offset = ppgtt->pd_offset;
691
	 *
-
 
692
	 * FIXME: We should try not to special case reset
-
 
693
	 */
-
 
694
	if (synchronous ||
505
	pd_offset /= 64; /* in cachelines, */
695
	    i915_reset_in_progress(&dev_priv->gpu_error)) {
-
 
696
		WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
-
 
697
		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
Line 506... Line 698...
506
	pd_offset <<= 16;
698
		I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
507
 
699
		POSTING_READ(RING_PP_DIR_BASE(ring));
-
 
700
		return 0;
508
	if (INTEL_INFO(dev)->gen == 6) {
701
	}
Line 509... Line 702...
509
		uint32_t ecochk, gab_ctl, ecobits;
702
 
-
 
703
	/* NB: TLBs must be flushed and invalidated before a switch */
510
 
704
	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Line -... Line 705...
-
 
705
	if (ret)
-
 
706
		return ret;
-
 
707
 
-
 
708
	ret = intel_ring_begin(ring, 6);
-
 
709
	if (ret)
511
		ecobits = I915_READ(GAC_ECO_BITS);
710
		return ret;
-
 
711
 
-
 
712
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
-
 
713
	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
-
 
714
	intel_ring_emit(ring, PP_DIR_DCLV_2G);
-
 
715
	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
-
 
716
	intel_ring_emit(ring, get_pd_offset(ppgtt));
-
 
717
	intel_ring_emit(ring, MI_NOOP);
-
 
718
	intel_ring_advance(ring);
-
 
719
 
-
 
720
	return 0;
-
 
721
}
-
 
722
 
-
 
723
static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
-
 
724
			  struct intel_engine_cs *ring,
-
 
725
			  bool synchronous)
-
 
726
{
-
 
727
	struct drm_device *dev = ppgtt->base.dev;
-
 
728
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
729
	int ret;
-
 
730
 
-
 
731
	/* If we're in reset, we can assume the GPU is sufficiently idle to
-
 
732
	 * manually frob these bits. Ideally we could use the ring functions,
-
 
733
	 * except our error handling makes it quite difficult (can't use
512
		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
734
	 * intel_ring_begin, ring->flush, or intel_ring_advance)
-
 
735
	 *
513
					 ECOBITS_PPGTT_CACHE64B);
736
	 * FIXME: We should try not to special case reset
-
 
737
	 */
-
 
738
	if (synchronous ||
-
 
739
	    i915_reset_in_progress(&dev_priv->gpu_error)) {
-
 
740
		WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
-
 
741
		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
-
 
742
		I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
-
 
743
		POSTING_READ(RING_PP_DIR_BASE(ring));
-
 
744
		return 0;
-
 
745
	}
-
 
746
 
-
 
747
	/* NB: TLBs must be flushed and invalidated before a switch */
-
 
748
	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
-
 
749
	if (ret)
-
 
750
		return ret;
-
 
751
 
-
 
752
	ret = intel_ring_begin(ring, 6);
-
 
753
	if (ret)
-
 
754
		return ret;
-
 
755
 
-
 
756
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
-
 
757
	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
-
 
758
	intel_ring_emit(ring, PP_DIR_DCLV_2G);
-
 
759
	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
-
 
760
	intel_ring_emit(ring, get_pd_offset(ppgtt));
-
 
761
	intel_ring_emit(ring, MI_NOOP);
-
 
762
	intel_ring_advance(ring);
-
 
763
 
-
 
764
	/* XXX: RCS is the only one to auto invalidate the TLBs? */
-
 
765
	if (ring->id != RCS) {
-
 
766
		ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
-
 
767
		if (ret)
-
 
768
			return ret;
-
 
769
	}
-
 
770
 
-
 
771
	return 0;
-
 
772
}
-
 
773
 
-
 
774
static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
-
 
775
			  struct intel_engine_cs *ring,
-
 
776
			  bool synchronous)
-
 
777
{
-
 
778
	struct drm_device *dev = ppgtt->base.dev;
-
 
779
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
780
 
-
 
781
	if (!synchronous)
-
 
782
		return 0;
-
 
783
 
-
 
784
	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
-
 
785
	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
-
 
786
 
-
 
787
	POSTING_READ(RING_PP_DIR_DCLV(ring));
-
 
788
 
-
 
789
	return 0;
-
 
790
}
-
 
791
 
-
 
792
static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
-
 
793
{
514
 
794
	struct drm_device *dev = ppgtt->base.dev;
-
 
795
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
796
	struct intel_engine_cs *ring;
-
 
797
	int j, ret;
-
 
798
 
-
 
799
	for_each_ring(ring, dev_priv, j) {
-
 
800
		I915_WRITE(RING_MODE_GEN7(ring),
-
 
801
			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-
 
802
 
-
 
803
		/* We promise to do a switch later with FULL PPGTT. If this is
-
 
804
		 * aliasing, this is the one and only switch we'll do */
-
 
805
		if (USES_FULL_PPGTT(dev))
-
 
806
			continue;
-
 
807
 
-
 
808
		ret = ppgtt->switch_mm(ppgtt, ring, true);
-
 
809
		if (ret)
515
		gab_ctl = I915_READ(GAB_CTL);
810
			goto err_out;
-
 
811
	}
-
 
812
 
-
 
813
	return 0;
-
 
814
 
-
 
815
err_out:
-
 
816
	for_each_ring(ring, dev_priv, j)
-
 
817
		I915_WRITE(RING_MODE_GEN7(ring),
-
 
818
			   _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
-
 
819
	return ret;
516
		I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
820
}
-
 
821
 
Line 517... Line 822...
517
 
822
static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
518
		ecochk = I915_READ(GAM_ECOCHK);
823
{
Line 519... Line 824...
519
		I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
824
	struct drm_device *dev = ppgtt->base.dev;
Line 531... Line 836...
531
		} else {
836
		} else {
532
			ecochk |= ECOCHK_PPGTT_LLC_IVB;
837
			ecochk |= ECOCHK_PPGTT_LLC_IVB;
533
			ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
838
			ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
534
		}
839
		}
535
		I915_WRITE(GAM_ECOCHK, ecochk);
840
		I915_WRITE(GAM_ECOCHK, ecochk);
536
		/* GFX_MODE is per-ring on gen7+ */
-
 
537
	}
-
 
Line 538... Line 841...
538
 
841
 
-
 
842
	for_each_ring(ring, dev_priv, i) {
539
	for_each_ring(ring, dev_priv, i) {
843
		int ret;
540
		if (INTEL_INFO(dev)->gen >= 7)
844
		/* GFX_MODE is per-ring on gen7+ */
541
			I915_WRITE(RING_MODE_GEN7(ring),
845
			I915_WRITE(RING_MODE_GEN7(ring),
Line -... Line 846...
-
 
846
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-
 
847
 
-
 
848
		/* We promise to do a switch later with FULL PPGTT. If this is
-
 
849
		 * aliasing, this is the one and only switch we'll do */
-
 
850
		if (USES_FULL_PPGTT(dev))
-
 
851
			continue;
-
 
852
 
-
 
853
		ret = ppgtt->switch_mm(ppgtt, ring, true);
-
 
854
		if (ret)
-
 
855
			return ret;
-
 
856
	}
-
 
857
 
-
 
858
	return 0;
-
 
859
}
-
 
860
 
-
 
861
static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
-
 
862
{
-
 
863
	struct drm_device *dev = ppgtt->base.dev;
-
 
864
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
865
	struct intel_engine_cs *ring;
-
 
866
	uint32_t ecochk, gab_ctl, ecobits;
-
 
867
	int i;
542
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
868
 
-
 
869
	ecobits = I915_READ(GAC_ECO_BITS);
-
 
870
	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
-
 
871
		   ECOBITS_PPGTT_CACHE64B);
-
 
872
 
-
 
873
	gab_ctl = I915_READ(GAB_CTL);
-
 
874
	I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
-
 
875
 
-
 
876
	ecochk = I915_READ(GAM_ECOCHK);
543
 
877
	I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
-
 
878
 
-
 
879
	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-
 
880
 
-
 
881
	for_each_ring(ring, dev_priv, i) {
-
 
882
		int ret = ppgtt->switch_mm(ppgtt, ring, true);
544
		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
883
		if (ret)
-
 
884
			return ret;
545
		I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
885
	}
546
	}
886
 
Line 547... Line 887...
547
	return 0;
887
	return 0;
548
}
888
}
549
 
889
 
550
/* PPGTT support for Sandybdrige/Gen6 and later */
890
/* PPGTT support for Sandybdrige/Gen6 and later */
551
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
891
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
552
				   unsigned first_entry,
892
				   uint64_t start,
553
				   unsigned num_entries,
893
				   uint64_t length,
554
				   bool use_scratch)
894
				   bool use_scratch)
555
{
895
{
-
 
896
	struct i915_hw_ppgtt *ppgtt =
-
 
897
		container_of(vm, struct i915_hw_ppgtt, base);
556
	struct i915_hw_ppgtt *ppgtt =
898
	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
557
		container_of(vm, struct i915_hw_ppgtt, base);
899
	unsigned first_entry = start >> PAGE_SHIFT;
558
	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
900
	unsigned num_entries = length >> PAGE_SHIFT;
Line 559... Line 901...
559
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
901
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
Line 560... Line 902...
560
	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
902
	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
Line 561... Line 903...
561
	unsigned last_pte, i;
903
	unsigned last_pte, i;
562
 
904
 
Line 585... Line 927...
585
    FreeKernelSpace(pt_vaddr);
927
    FreeKernelSpace(pt_vaddr);
586
}
928
}
Line 587... Line 929...
587
 
929
 
588
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
930
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
589
				      struct sg_table *pages,
931
				      struct sg_table *pages,
590
				      unsigned first_entry,
932
				      uint64_t start,
591
				      enum i915_cache_level cache_level)
933
				      enum i915_cache_level cache_level, u32 flags)
592
{
934
{
593
	struct i915_hw_ppgtt *ppgtt =
935
	struct i915_hw_ppgtt *ppgtt =
594
		container_of(vm, struct i915_hw_ppgtt, base);
936
		container_of(vm, struct i915_hw_ppgtt, base);
-
 
937
	gen6_gtt_pte_t *pt_vaddr;
595
	gen6_gtt_pte_t *pt_vaddr;
938
	unsigned first_entry = start >> PAGE_SHIFT;
596
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
939
	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
597
	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
940
	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
598
	struct sg_page_iter sg_iter;
-
 
599
	dma_addr_t page_addr;
-
 
Line 600... Line 941...
600
 
941
	struct sg_page_iter sg_iter;
Line 601... Line 942...
601
 
942
 
602
    pt_vaddr = AllocKernelSpace(4096);
943
    pt_vaddr = AllocKernelSpace(4096);
Line 607... Line 948...
607
    MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
948
    MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
608
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
949
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
Line 609... Line 950...
609
 
950
 
610
		pt_vaddr[act_pte] =
951
		pt_vaddr[act_pte] =
611
			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
952
			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
-
 
953
				       cache_level, true, flags);
612
				       cache_level, true);
954
 
613
		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
955
		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
614
			act_pt++;
956
			act_pt++;
615
    		MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
957
    		MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3);
616
			act_pte = 0;
-
 
617
 
958
			act_pte = 0;
618
			}
959
			}
619
		}
960
		}
620
    FreeKernelSpace(pt_vaddr);
961
    FreeKernelSpace(pt_vaddr);
Line 621... Line 962...
621
}
962
}
622
 
963
 
623
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
-
 
624
{
-
 
625
	struct i915_hw_ppgtt *ppgtt =
964
static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
Line 626... Line -...
626
		container_of(vm, struct i915_hw_ppgtt, base);
-
 
627
	int i;
-
 
628
 
965
{
629
	drm_mm_takedown(&ppgtt->base.mm);
966
	int i;
630
 
967
 
631
	if (ppgtt->pt_dma_addr) {
968
	if (ppgtt->pt_dma_addr) {
632
		for (i = 0; i < ppgtt->num_pd_entries; i++)
969
		for (i = 0; i < ppgtt->num_pd_entries; i++)
633
			pci_unmap_page(ppgtt->base.dev->pdev,
970
			pci_unmap_page(ppgtt->base.dev->pdev,
-
 
971
				       ppgtt->pt_dma_addr[i],
-
 
972
				       4096, PCI_DMA_BIDIRECTIONAL);
-
 
973
	}
-
 
974
}
-
 
975
 
Line 634... Line 976...
634
				       ppgtt->pt_dma_addr[i],
976
static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
635
				       4096, PCI_DMA_BIDIRECTIONAL);
977
{
636
	}
978
	int i;
637
 
979
 
638
	kfree(ppgtt->pt_dma_addr);
-
 
639
	for (i = 0; i < ppgtt->num_pd_entries; i++)
980
	kfree(ppgtt->pt_dma_addr);
Line -... Line 981...
-
 
981
	for (i = 0; i < ppgtt->num_pd_entries; i++)
-
 
982
		__free_page(ppgtt->pt_pages[i]);
-
 
983
	kfree(ppgtt->pt_pages);
-
 
984
}
-
 
985
 
-
 
986
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
-
 
987
{
-
 
988
	struct i915_hw_ppgtt *ppgtt =
-
 
989
		container_of(vm, struct i915_hw_ppgtt, base);
-
 
990
 
-
 
991
	list_del(&vm->global_link);
-
 
992
	drm_mm_takedown(&ppgtt->base.mm);
-
 
993
	drm_mm_remove_node(&ppgtt->node);
640
		__free_page(ppgtt->pt_pages[i]);
994
 
641
	kfree(ppgtt->pt_pages);
995
	gen6_ppgtt_unmap_pages(ppgtt);
642
	kfree(ppgtt);
996
	gen6_ppgtt_free(ppgtt);
643
}
997
}
644
 
998
 
645
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
999
static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
646
{
-
 
Line 647... Line 1000...
647
	struct drm_device *dev = ppgtt->base.dev;
1000
{
-
 
1001
	struct drm_device *dev = ppgtt->base.dev;
648
	struct drm_i915_private *dev_priv = dev->dev_private;
1002
	struct drm_i915_private *dev_priv = dev->dev_private;
649
	unsigned first_pd_entry_in_global_pt;
1003
	bool retried = false;
-
 
1004
	int ret;
-
 
1005
 
650
	int i;
1006
	/* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
-
 
1007
	 * allocator works in address space sizes, so it's multiplied by page
-
 
1008
	 * size. We allocate at the top of the GTT to avoid fragmentation.
-
 
1009
	 */
-
 
1010
	BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
-
 
1011
alloc:
-
 
1012
	ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
-
 
1013
						  &ppgtt->node, GEN6_PD_SIZE,
-
 
1014
						  GEN6_PD_ALIGN, 0,
-
 
1015
						  0, dev_priv->gtt.base.total,
-
 
1016
						  DRM_MM_TOPDOWN);
-
 
1017
	if (ret == -ENOSPC && !retried) {
-
 
1018
		ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
-
 
1019
					       GEN6_PD_SIZE, GEN6_PD_ALIGN,
-
 
1020
					       I915_CACHE_NONE,
-
 
1021
					       0, dev_priv->gtt.base.total,
-
 
1022
					       0);
-
 
1023
		if (ret)
-
 
1024
			return ret;
-
 
1025
 
Line 651... Line -...
651
	int ret = -ENOMEM;
-
 
652
 
1026
		retried = true;
653
	/* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
1027
		goto alloc;
654
	 * entries. For aliasing ppgtt support we just steal them at the end for
-
 
-
 
1028
	}
-
 
1029
 
655
	 * now. */
1030
	if (ppgtt->node.start < dev_priv->gtt.mappable_end)
656
       first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
-
 
657
 
-
 
-
 
1031
		DRM_DEBUG("Forced to use aperture for PDEs\n");
658
	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
1032
 
659
	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
-
 
-
 
1033
	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
660
	ppgtt->enable = gen6_ppgtt_enable;
1034
	return ret;
661
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1035
}
-
 
1036
 
662
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1037
static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
663
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
1038
{
Line 664... Line 1039...
664
	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
1039
	int i;
665
	ppgtt->base.start = 0;
1040
 
666
	ppgtt->base.total = GEN6_PPGTT_PD_ENTRIES * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
1041
	ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
-
 
1042
				  GFP_KERNEL);
667
	ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
1043
 
-
 
1044
	if (!ppgtt->pt_pages)
-
 
1045
		return -ENOMEM;
-
 
1046
 
-
 
1047
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-
 
1048
		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
-
 
1049
		if (!ppgtt->pt_pages[i]) {
-
 
1050
			gen6_ppgtt_free(ppgtt);
-
 
1051
			return -ENOMEM;
-
 
1052
		}
-
 
1053
	}
-
 
1054
 
-
 
1055
	return 0;
-
 
1056
}
-
 
1057
 
-
 
1058
static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
-
 
1059
{
-
 
1060
	int ret;
-
 
1061
 
668
				  GFP_KERNEL);
1062
	ret = gen6_ppgtt_allocate_page_directories(ppgtt);
Line 669... Line 1063...
669
	if (!ppgtt->pt_pages)
1063
	if (ret)
670
		return -ENOMEM;
1064
		return ret;
671
 
1065
 
-
 
1066
	ret = gen6_ppgtt_allocate_page_tables(ppgtt);
672
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
1067
	if (ret) {
-
 
1068
		drm_mm_remove_node(&ppgtt->node);
-
 
1069
		return ret;
-
 
1070
	}
-
 
1071
 
-
 
1072
	ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
-
 
1073
				     GFP_KERNEL);
-
 
1074
	if (!ppgtt->pt_dma_addr) {
-
 
1075
		drm_mm_remove_node(&ppgtt->node);
-
 
1076
		gen6_ppgtt_free(ppgtt);
-
 
1077
		return -ENOMEM;
Line 673... Line 1078...
673
		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
1078
	}
674
		if (!ppgtt->pt_pages[i])
1079
 
Line 675... Line 1080...
675
			goto err_pt_alloc;
1080
	return 0;
676
	}
1081
}
Line -... Line 1082...
-
 
1082
 
-
 
1083
static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
-
 
1084
{
-
 
1085
	struct drm_device *dev = ppgtt->base.dev;
-
 
1086
	int i;
677
 
1087
 
678
	ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
1088
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
Line 679... Line 1089...
679
					     GFP_KERNEL);
1089
		dma_addr_t pt_addr;
680
    if (!ppgtt->pt_dma_addr)
-
 
-
 
1090
 
Line 681... Line 1091...
681
        goto err_pt_alloc;
1091
		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
-
 
1092
				       PCI_DMA_BIDIRECTIONAL);
-
 
1093
 
-
 
1094
//       if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
-
 
1095
//           gen6_ppgtt_unmap_pages(ppgtt);
Line -... Line 1096...
-
 
1096
//           return -EIO;
-
 
1097
//       }
-
 
1098
 
-
 
1099
		ppgtt->pt_dma_addr[i] = pt_addr;
-
 
1100
	}
-
 
1101
 
-
 
1102
	return 0;
-
 
1103
}
-
 
1104
 
-
 
1105
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
-
 
1106
{
682
 
1107
	struct drm_device *dev = ppgtt->base.dev;
Line 683... Line -...
683
    for (i = 0; i < ppgtt->num_pd_entries; i++) {
-
 
684
        dma_addr_t pt_addr;
-
 
685
 
-
 
686
		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
-
 
687
					       PCI_DMA_BIDIRECTIONAL);
-
 
688
 
-
 
689
        ppgtt->pt_dma_addr[i] = pt_addr;
-
 
690
    }
1108
	struct drm_i915_private *dev_priv = dev->dev_private;
691
 
-
 
692
	ppgtt->base.clear_range(&ppgtt->base, 0,
1109
	int ret;
693
				ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true);
-
 
694
 
-
 
695
	ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
1110
 
Line -... Line 1111...
-
 
1111
	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
-
 
1112
	if (IS_GEN6(dev)) {
-
 
1113
	ppgtt->enable = gen6_ppgtt_enable;
696
 
1114
		ppgtt->switch_mm = gen6_mm_switch;
697
	return 0;
1115
	} else if (IS_HASWELL(dev)) {
Line -... Line 1116...
-
 
1116
		ppgtt->enable = gen7_ppgtt_enable;
-
 
1117
		ppgtt->switch_mm = hsw_mm_switch;
-
 
1118
	} else if (IS_GEN7(dev)) {
-
 
1119
		ppgtt->enable = gen7_ppgtt_enable;
-
 
1120
		ppgtt->switch_mm = gen7_mm_switch;
-
 
1121
	} else
-
 
1122
		BUG();
-
 
1123
 
-
 
1124
	ret = gen6_ppgtt_alloc(ppgtt);
-
 
1125
	if (ret)
-
 
1126
		return ret;
-
 
1127
 
-
 
1128
	ret = gen6_ppgtt_setup_page_tables(ppgtt);
-
 
1129
	if (ret) {
-
 
1130
		gen6_ppgtt_free(ppgtt);
-
 
1131
		return ret;
-
 
1132
	}
-
 
1133
 
-
 
1134
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
698
 
1135
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
699
err_pd_pin:
1136
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
700
	if (ppgtt->pt_dma_addr) {
1137
	ppgtt->base.start = 0;
701
		for (i--; i >= 0; i--)
-
 
702
			pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
1138
	ppgtt->base.total =  ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
703
				       4096, PCI_DMA_BIDIRECTIONAL);
-
 
704
	}
-
 
705
err_pt_alloc:
-
 
706
	kfree(ppgtt->pt_dma_addr);
-
 
Line 707... Line 1139...
707
	for (i = 0; i < ppgtt->num_pd_entries; i++) {
1139
//	ppgtt->debug_dump = gen6_dump_ppgtt;
-
 
1140
 
Line 708... Line 1141...
708
		if (ppgtt->pt_pages[i])
1141
	ppgtt->pd_offset =
709
			__free_page(ppgtt->pt_pages[i]);
1142
		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
710
	}
1143
 
711
	kfree(ppgtt->pt_pages);
1144
	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
712
 
1145
 
713
	return ret;
1146
	DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
Line 714... Line 1147...
714
}
1147
			 ppgtt->node.size >> 20,
715
 
1148
			 ppgtt->node.start / PAGE_SIZE);
716
static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
-
 
717
{
1149
 
718
	struct drm_i915_private *dev_priv = dev->dev_private;
1150
	return 0;
719
	struct i915_hw_ppgtt *ppgtt;
1151
}
-
 
1152
 
-
 
1153
int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
-
 
1154
{
-
 
1155
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
1156
	int ret = 0;
-
 
1157
 
720
	int ret;
1158
	ppgtt->base.dev = dev;
Line 721... Line 1159...
721
 
1159
	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
722
	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1160
 
Line -... Line 1161...
-
 
1161
	if (INTEL_INFO(dev)->gen < 8)
723
	if (!ppgtt)
1162
		ret = gen6_ppgtt_init(ppgtt);
-
 
1163
	else if (IS_GEN8(dev))
-
 
1164
		ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
724
		return -ENOMEM;
1165
	else
725
 
1166
		BUG();
726
	ppgtt->base.dev = dev;
-
 
727
 
-
 
728
	if (INTEL_INFO(dev)->gen < 8)
1167
 
729
	ret = gen6_ppgtt_init(ppgtt);
1168
	if (!ret) {
Line 730... Line 1169...
730
	else if (IS_GEN8(dev))
1169
		struct drm_i915_private *dev_priv = dev->dev_private;
731
		ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
1170
		kref_init(&ppgtt->ref);
732
	else
1171
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
Line 733... Line 1172...
733
		BUG();
1172
			    ppgtt->base.total);
734
 
-
 
735
	if (ret)
-
 
736
        kfree(ppgtt);
1173
		i915_init_vm(dev_priv, &ppgtt->base);
737
	else {
1174
		if (INTEL_INFO(dev)->gen < 8) {
738
		dev_priv->mm.aliasing_ppgtt = ppgtt;
-
 
739
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1175
			gen6_write_pdes(ppgtt);
740
			    ppgtt->base.total);
-
 
741
	}
-
 
742
 
-
 
743
	return ret;
-
 
744
}
-
 
745
 
-
 
746
void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
-
 
747
{
1176
			DRM_DEBUG("Adding PPGTT at offset %x\n",
748
	struct drm_i915_private *dev_priv = dev->dev_private;
1177
				  ppgtt->pd_offset << 10);
749
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1178
		}
Line 750... Line 1179...
750
 
1179
	}
751
	if (!ppgtt)
1180
 
Line 812... Line 1241...
812
}
1241
}
Line 813... Line 1242...
813
 
1242
 
814
void i915_check_and_clear_faults(struct drm_device *dev)
1243
void i915_check_and_clear_faults(struct drm_device *dev)
815
{
1244
{
816
	struct drm_i915_private *dev_priv = dev->dev_private;
1245
	struct drm_i915_private *dev_priv = dev->dev_private;
817
	struct intel_ring_buffer *ring;
1246
	struct intel_engine_cs *ring;
Line 818... Line 1247...
818
	int i;
1247
	int i;
819
 
1248
 
Line 851... Line 1280...
851
		return;
1280
		return;
Line 852... Line 1281...
852
 
1281
 
Line 853... Line 1282...
853
	i915_check_and_clear_faults(dev);
1282
	i915_check_and_clear_faults(dev);
854
 
1283
 
855
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1284
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
856
				       dev_priv->gtt.base.start / PAGE_SIZE,
1285
				       dev_priv->gtt.base.start,
857
				       dev_priv->gtt.base.total / PAGE_SIZE,
1286
				       dev_priv->gtt.base.total,
Line 858... Line 1287...
858
				       false);
1287
				       true);
859
}
1288
}
860
 
1289
 
861
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1290
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
-
 
1291
{
Line 862... Line 1292...
862
{
1292
	struct drm_i915_private *dev_priv = dev->dev_private;
Line 863... Line 1293...
863
	struct drm_i915_private *dev_priv = dev->dev_private;
1293
	struct drm_i915_gem_object *obj;
864
	struct drm_i915_gem_object *obj;
1294
	struct i915_address_space *vm;
865
 
1295
 
866
	i915_check_and_clear_faults(dev);
1296
	i915_check_and_clear_faults(dev);
867
 
1297
 
Line 868... Line 1298...
868
	/* First fill our portion of the GTT with scratch pages */
1298
	/* First fill our portion of the GTT with scratch pages */
-
 
1299
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
-
 
1300
				       dev_priv->gtt.base.start,
-
 
1301
				       dev_priv->gtt.base.total,
-
 
1302
				       true);
-
 
1303
 
869
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1304
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-
 
1305
		struct i915_vma *vma = i915_gem_obj_to_vma(obj,
-
 
1306
							   &dev_priv->gtt.base);
-
 
1307
		if (!vma)
-
 
1308
			continue;
-
 
1309
 
870
				       dev_priv->gtt.base.start / PAGE_SIZE,
1310
		i915_gem_clflush_object(obj, obj->pin_display);
-
 
1311
		/* The bind_vma code tries to be smart about tracking mappings.
-
 
1312
		 * Unfortunately above, we've just wiped out the mappings
-
 
1313
		 * without telling our object about it. So we need to fake it.
-
 
1314
		 */
-
 
1315
		obj->has_global_gtt_mapping = 0;
-
 
1316
		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
-
 
1317
	}
-
 
1318
 
-
 
1319
 
-
 
1320
	if (INTEL_INFO(dev)->gen >= 8) {
-
 
1321
		if (IS_CHERRYVIEW(dev))
-
 
1322
			chv_setup_private_ppat(dev_priv);
-
 
1323
		else
-
 
1324
			bdw_setup_private_ppat(dev_priv);
-
 
1325
 
-
 
1326
		return;
-
 
1327
	}
-
 
1328
 
-
 
1329
	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
-
 
1330
		/* TODO: Perhaps it shouldn't be gen6 specific */
-
 
1331
		if (i915_is_ggtt(vm)) {
871
				       dev_priv->gtt.base.total / PAGE_SIZE,
1332
			if (dev_priv->mm.aliasing_ppgtt)
Line 872... Line 1333...
872
				       true);
1333
				gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
873
 
1334
			continue;
Line 902... Line 1363...
902
#endif
1363
#endif
903
}
1364
}
Line 904... Line 1365...
904
 
1365
 
905
static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1366
static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
906
				     struct sg_table *st,
1367
				     struct sg_table *st,
907
				     unsigned int first_entry,
1368
				     uint64_t start,
908
				     enum i915_cache_level level)
1369
				     enum i915_cache_level level, u32 unused)
909
{
1370
{
-
 
1371
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
910
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
1372
	unsigned first_entry = start >> PAGE_SHIFT;
911
	gen8_gtt_pte_t __iomem *gtt_entries =
1373
	gen8_gtt_pte_t __iomem *gtt_entries =
912
		(gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1374
		(gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
913
	int i = 0;
1375
	int i = 0;
914
	struct sg_page_iter sg_iter;
1376
	struct sg_page_iter sg_iter;
Line 915... Line 1377...
915
	dma_addr_t addr;
1377
	dma_addr_t addr = 0; /* shut up gcc */
916
 
1378
 
917
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1379
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
918
		addr = sg_dma_address(sg_iter.sg) +
1380
		addr = sg_dma_address(sg_iter.sg) +
Line 947... Line 1409...
947
 * within the global GTT as well as accessible by the GPU through the GMADR
1409
 * within the global GTT as well as accessible by the GPU through the GMADR
948
 * mapped BAR (dev_priv->mm.gtt->gtt).
1410
 * mapped BAR (dev_priv->mm.gtt->gtt).
949
 */
1411
 */
950
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1412
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
951
				     struct sg_table *st,
1413
				     struct sg_table *st,
952
				     unsigned int first_entry,
1414
				     uint64_t start,
953
				  enum i915_cache_level level)
1415
				     enum i915_cache_level level, u32 flags)
954
{
1416
{
955
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
1417
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
-
 
1418
	unsigned first_entry = start >> PAGE_SHIFT;
956
	gen6_gtt_pte_t __iomem *gtt_entries =
1419
	gen6_gtt_pte_t __iomem *gtt_entries =
957
		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1420
		(gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
958
	int i = 0;
1421
	int i = 0;
959
	struct sg_page_iter sg_iter;
1422
	struct sg_page_iter sg_iter;
960
	dma_addr_t addr;
1423
	dma_addr_t addr = 0;
Line 961... Line 1424...
961
 
1424
 
962
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1425
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
963
		addr = sg_page_iter_dma_address(&sg_iter);
1426
		addr = sg_page_iter_dma_address(&sg_iter);
964
		iowrite32(vm->pte_encode(addr, level, true), >t_entries[i]);
1427
		iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]);
965
			i++;
1428
			i++;
Line 966... Line 1429...
966
		}
1429
		}
967
 
1430
 
968
	/* XXX: This serves as a posting read to make sure that the PTE has
1431
	/* XXX: This serves as a posting read to make sure that the PTE has
969
	 * actually been updated. There is some concern that even though
1432
	 * actually been updated. There is some concern that even though
970
	 * registers and PTEs are within the same BAR that they are potentially
1433
	 * registers and PTEs are within the same BAR that they are potentially
971
	 * of NUMA access patterns. Therefore, even with the way we assume
1434
	 * of NUMA access patterns. Therefore, even with the way we assume
972
	 * hardware should work, we must keep this posting read for paranoia.
1435
	 * hardware should work, we must keep this posting read for paranoia.
973
	 */
1436
	 */
974
	if (i != 0)
1437
	if (i != 0) {
-
 
1438
		unsigned long gtt = readl(>t_entries[i-1]);
Line 975... Line 1439...
975
		WARN_ON(readl(>t_entries[i-1]) !=
1439
		WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
976
			vm->pte_encode(addr, level, true));
1440
	}
977
 
1441
 
978
	/* This next bit makes the above posting read even more important. We
1442
	/* This next bit makes the above posting read even more important. We
979
	 * want to flush the TLBs only after we're certain all the PTE updates
1443
	 * want to flush the TLBs only after we're certain all the PTE updates
980
	 * have finished.
1444
	 * have finished.
981
	 */
1445
	 */
Line 982... Line 1446...
982
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1446
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
983
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
1447
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
984
}
1448
}
985
 
1449
 
986
static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1450
static void gen8_ggtt_clear_range(struct i915_address_space *vm,
987
				  unsigned int first_entry,
1451
				  uint64_t start,
-
 
1452
				  uint64_t length,
-
 
1453
				  bool use_scratch)
988
				  unsigned int num_entries,
1454
{
989
				  bool use_scratch)
1455
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
990
{
1456
	unsigned first_entry = start >> PAGE_SHIFT;
991
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
1457
	unsigned num_entries = length >> PAGE_SHIFT;
Line 1006... Line 1472...
1006
		gen8_set_pte(>t_base[i], scratch_pte);
1472
		gen8_set_pte(>t_base[i], scratch_pte);
1007
	readl(gtt_base);
1473
	readl(gtt_base);
1008
}
1474
}
Line 1009... Line 1475...
1009
 
1475
 
1010
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1476
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1011
				  unsigned int first_entry,
1477
				  uint64_t start,
1012
				  unsigned int num_entries,
1478
				  uint64_t length,
1013
				  bool use_scratch)
1479
				  bool use_scratch)
1014
{
1480
{
-
 
1481
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
-
 
1482
	unsigned first_entry = start >> PAGE_SHIFT;
1015
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
1483
	unsigned num_entries = length >> PAGE_SHIFT;
1016
	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1484
	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1017
		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1485
		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1018
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1486
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
Line 1019... Line 1487...
1019
	int i;
1487
	int i;
1020
 
1488
 
1021
	if (WARN(num_entries > max_entries,
1489
	if (WARN(num_entries > max_entries,
1022
		 "First entry = %d; Num entries = %d (max=%d)\n",
1490
		 "First entry = %d; Num entries = %d (max=%d)\n",
Line 1023... Line 1491...
1023
		 first_entry, num_entries, max_entries))
1491
		 first_entry, num_entries, max_entries))
Line 1024... Line 1492...
1024
        num_entries = max_entries;
1492
        num_entries = max_entries;
1025
 
1493
 
1026
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch);
1494
	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
1027
 
1495
 
Line -... Line 1496...
-
 
1496
	for (i = 0; i < num_entries; i++)
1028
	for (i = 0; i < num_entries; i++)
1497
		iowrite32(scratch_pte, >t_base[i]);
1029
		iowrite32(scratch_pte, >t_base[i]);
1498
	readl(gtt_base);
1030
	readl(gtt_base);
1499
}
1031
}
-
 
1032
 
1500
 
-
 
1501
 
1033
static void i915_ggtt_insert_entries(struct i915_address_space *vm,
1502
static void i915_ggtt_bind_vma(struct i915_vma *vma,
1034
				     struct sg_table *st,
1503
			       enum i915_cache_level cache_level,
Line -... Line 1504...
-
 
1504
			       u32 unused)
1035
				     unsigned int pg_start,
1505
{
1036
				     enum i915_cache_level cache_level)
-
 
-
 
1506
	const unsigned long entry = vma->node.start >> PAGE_SHIFT;
1037
{
1507
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
Line 1038... Line 1508...
1038
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1508
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1039
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1509
 
1040
 
1510
	BUG_ON(!i915_is_ggtt(vma->vm));
1041
	intel_gtt_insert_sg_entries(st, pg_start, flags);
1511
	intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
1042
 
1512
	vma->obj->has_global_gtt_mapping = 1;
-
 
1513
}
-
 
1514
 
1043
}
1515
static void i915_ggtt_clear_range(struct i915_address_space *vm,
1044
 
1516
				  uint64_t start,
Line -... Line 1517...
-
 
1517
				  uint64_t length,
-
 
1518
				  bool unused)
-
 
1519
{
-
 
1520
	unsigned first_entry = start >> PAGE_SHIFT;
-
 
1521
	unsigned num_entries = length >> PAGE_SHIFT;
-
 
1522
	intel_gtt_clear_range(first_entry, num_entries);
-
 
1523
}
-
 
1524
 
-
 
1525
static void i915_ggtt_unbind_vma(struct i915_vma *vma)
Line 1045... Line 1526...
1045
static void i915_ggtt_clear_range(struct i915_address_space *vm,
1526
{
1046
				  unsigned int first_entry,
1527
	const unsigned int first = vma->node.start >> PAGE_SHIFT;
-
 
1528
	const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
1047
				  unsigned int num_entries,
1529
 
1048
				  bool unused)
1530
	BUG_ON(!i915_is_ggtt(vma->vm));
1049
{
1531
	vma->obj->has_global_gtt_mapping = 0;
1050
	intel_gtt_clear_range(first_entry, num_entries);
1532
	intel_gtt_clear_range(first, size);
1051
}
-
 
1052
 
-
 
1053
 
-
 
1054
void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
-
 
Line -... Line 1533...
-
 
1533
}
-
 
1534
 
-
 
1535
static void ggtt_bind_vma(struct i915_vma *vma,
-
 
1536
			  enum i915_cache_level cache_level,
-
 
1537
			  u32 flags)
-
 
1538
{
-
 
1539
	struct drm_device *dev = vma->vm->dev;
-
 
1540
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
1541
	struct drm_i915_gem_object *obj = vma->obj;
-
 
1542
 
-
 
1543
	/* Currently applicable only to VLV */
-
 
1544
	if (obj->gt_ro)
-
 
1545
		flags |= PTE_READ_ONLY;
-
 
1546
 
-
 
1547
	/* If there is no aliasing PPGTT, or the caller needs a global mapping,
-
 
1548
	 * or we have a global mapping already but the cacheability flags have
-
 
1549
	 * changed, set the global PTEs.
-
 
1550
	 *
-
 
1551
	 * If there is an aliasing PPGTT it is anecdotally faster, so use that
-
 
1552
	 * instead if none of the above hold true.
-
 
1553
	 *
1055
				enum i915_cache_level cache_level)
1554
	 * NB: A global mapping should only be needed for special regions like
1056
{
1555
	 * "gtt mappable", SNB errata, or if specified via special execbuf
-
 
1556
	 * flags. At all other times, the GPU will use the aliasing PPGTT.
Line -... Line 1557...
-
 
1557
	 */
-
 
1558
	if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
-
 
1559
		if (!obj->has_global_gtt_mapping ||
-
 
1560
		    (cache_level != obj->cache_level)) {
-
 
1561
			vma->vm->insert_entries(vma->vm, obj->pages,
-
 
1562
						vma->node.start,
-
 
1563
						cache_level, flags);
-
 
1564
			obj->has_global_gtt_mapping = 1;
-
 
1565
		}
-
 
1566
	}
-
 
1567
 
-
 
1568
	if (dev_priv->mm.aliasing_ppgtt &&
1057
	struct drm_device *dev = obj->base.dev;
1569
	    (!obj->has_aliasing_ppgtt_mapping ||
1058
	struct drm_i915_private *dev_priv = dev->dev_private;
1570
	     (cache_level != obj->cache_level))) {
1059
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
1571
		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1060
 
1572
		appgtt->base.insert_entries(&appgtt->base,
1061
	dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
1573
					    vma->obj->pages,
Line -... Line 1574...
-
 
1574
					    vma->node.start,
1062
					  entry,
1575
					    cache_level, flags);
1063
					 cache_level);
1576
		vma->obj->has_aliasing_ppgtt_mapping = 1;
1064
 
1577
	}
1065
	obj->has_global_gtt_mapping = 1;
1578
}
1066
}
-
 
1067
 
1579
 
1068
void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
1580
static void ggtt_unbind_vma(struct i915_vma *vma)
Line -... Line 1581...
-
 
1581
{
-
 
1582
	struct drm_device *dev = vma->vm->dev;
-
 
1583
	struct drm_i915_private *dev_priv = dev->dev_private;
-
 
1584
	struct drm_i915_gem_object *obj = vma->obj;
-
 
1585
 
-
 
1586
	if (obj->has_global_gtt_mapping) {
-
 
1587
		vma->vm->clear_range(vma->vm,
-
 
1588
				     vma->node.start,
-
 
1589
				     obj->base.size,
-
 
1590
				       true);
1069
{
1591
		obj->has_global_gtt_mapping = 0;
1070
	struct drm_device *dev = obj->base.dev;
1592
	}
1071
	struct drm_i915_private *dev_priv = dev->dev_private;
1593
 
1072
	const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
1594
	if (obj->has_aliasing_ppgtt_mapping) {
1073
 
1595
		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
Line 1156... Line 1678...
1156
	dev_priv->gtt.base.start = start;
1678
	dev_priv->gtt.base.start = start;
1157
	dev_priv->gtt.base.total = end - start;
1679
	dev_priv->gtt.base.total = end - start;
Line 1158... Line 1680...
1158
 
1680
 
1159
	/* Clear any non-preallocated blocks */
1681
	/* Clear any non-preallocated blocks */
1160
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
-
 
1161
		const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
1682
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
1162
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1683
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1163
			      hole_start, hole_end);
1684
			      hole_start, hole_end);
-
 
1685
		ggtt_vm->clear_range(ggtt_vm, hole_start,
1164
		ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true);
1686
				     hole_end - hole_start, true);
Line 1165... Line 1687...
1165
	}
1687
	}
1166
 
1688
 
1167
	/* And finally clear the reserved guard page */
-
 
1168
	ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true);
-
 
1169
}
-
 
1170
 
-
 
1171
static bool
-
 
1172
intel_enable_ppgtt(struct drm_device *dev)
-
 
1173
{
-
 
1174
	if (i915_enable_ppgtt >= 0)
-
 
1175
		return i915_enable_ppgtt;
-
 
1176
 
-
 
1177
#ifdef CONFIG_INTEL_IOMMU
-
 
1178
	/* Disable ppgtt on SNB if VT-d is on. */
-
 
1179
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
-
 
1180
		return false;
-
 
1181
#endif
-
 
1182
 
1689
	/* And finally clear the reserved guard page */
Line 1183... Line 1690...
1183
	return true;
1690
	ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
1184
}
1691
}
1185
 
1692
 
1186
void i915_gem_init_global_gtt(struct drm_device *dev)
1693
void i915_gem_init_global_gtt(struct drm_device *dev)
Line 1187... Line 1694...
1187
{
1694
{
1188
	struct drm_i915_private *dev_priv = dev->dev_private;
1695
	struct drm_i915_private *dev_priv = dev->dev_private;
Line 1189... Line -...
1189
	unsigned long gtt_size, mappable_size;
-
 
1190
 
-
 
1191
	gtt_size = dev_priv->gtt.base.total;
-
 
1192
	mappable_size = dev_priv->gtt.mappable_end;
-
 
1193
 
-
 
1194
	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
-
 
1195
		int ret;
-
 
1196
 
-
 
1197
		if (INTEL_INFO(dev)->gen <= 7) {
-
 
1198
		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
-
 
1199
		 * aperture accordingly when using aliasing ppgtt. */
-
 
1200
			gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
-
 
1201
		}
-
 
1202
 
-
 
1203
		i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
-
 
1204
 
-
 
1205
		ret = i915_gem_init_aliasing_ppgtt(dev);
-
 
1206
		if (!ret)
-
 
1207
			return;
-
 
1208
 
-
 
1209
		DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
1696
	unsigned long gtt_size, mappable_size;
1210
		drm_mm_takedown(&dev_priv->gtt.base.mm);
1697
 
Line 1211... Line 1698...
1211
		if (INTEL_INFO(dev)->gen < 8)
1698
	gtt_size = dev_priv->gtt.base.total;
1212
			gtt_size += GEN6_PPGTT_PD_ENTRIES*PAGE_SIZE;
1699
	mappable_size = dev_priv->gtt.mappable_end;
Line 1263... Line 1750...
1263
{
1750
{
1264
	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1751
	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1265
	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1752
	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1266
	if (bdw_gmch_ctl)
1753
	if (bdw_gmch_ctl)
1267
		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1754
		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
-
 
1755
 
-
 
1756
#ifdef CONFIG_X86_32
-
 
1757
	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
1268
	if (bdw_gmch_ctl > 4) {
1758
	if (bdw_gmch_ctl > 4)
1269
		WARN_ON(!i915_preliminary_hw_support);
-
 
1270
		return 4<<20;
1759
		bdw_gmch_ctl = 4;
1271
	}
1760
#endif
Line 1272... Line 1761...
1272
 
1761
 
1273
	return bdw_gmch_ctl << 20;
1762
	return bdw_gmch_ctl << 20;
Line -... Line 1763...
-
 
1763
}
-
 
1764
 
-
 
1765
static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
-
 
1766
{
-
 
1767
	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
-
 
1768
	gmch_ctrl &= SNB_GMCH_GGMS_MASK;
-
 
1769
 
-
 
1770
	if (gmch_ctrl)
-
 
1771
		return 1 << (20 + gmch_ctrl);
-
 
1772
 
-
 
1773
	return 0;
1274
}
1774
}
1275
 
1775
 
1276
static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1776
static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1277
{
1777
{
1278
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1778
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
Line 1285... Line 1785...
1285
	bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1785
	bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1286
	bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1786
	bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1287
	return bdw_gmch_ctl << 25; /* 32 MB units */
1787
	return bdw_gmch_ctl << 25; /* 32 MB units */
1288
}
1788
}
Line -... Line 1789...
-
 
1789
 
-
 
1790
static size_t chv_get_stolen_size(u16 gmch_ctrl)
-
 
1791
{
-
 
1792
	gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
-
 
1793
	gmch_ctrl &= SNB_GMCH_GMS_MASK;
-
 
1794
 
-
 
1795
	/*
-
 
1796
	 * 0x0  to 0x10: 32MB increments starting at 0MB
-
 
1797
	 * 0x11 to 0x16: 4MB increments starting at 8MB
-
 
1798
	 * 0x17 to 0x1d: 4MB increments start at 36MB
-
 
1799
	 */
-
 
1800
	if (gmch_ctrl < 0x11)
-
 
1801
		return gmch_ctrl << 25;
-
 
1802
	else if (gmch_ctrl < 0x17)
-
 
1803
		return (gmch_ctrl - 0x11 + 2) << 22;
-
 
1804
	else
-
 
1805
		return (gmch_ctrl - 0x17 + 9) << 22;
-
 
1806
}
1289
 
1807
 
1290
static int ggtt_probe_common(struct drm_device *dev,
1808
static int ggtt_probe_common(struct drm_device *dev,
1291
			     size_t gtt_size)
1809
			     size_t gtt_size)
1292
{
1810
{
1293
	struct drm_i915_private *dev_priv = dev->dev_private;
1811
	struct drm_i915_private *dev_priv = dev->dev_private;
Line 1315... Line 1833...
1315
}
1833
}
Line 1316... Line 1834...
1316
 
1834
 
1317
/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
1835
/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
1318
 * bits. When using advanced contexts each context stores its own PAT, but
1836
 * bits. When using advanced contexts each context stores its own PAT, but
1319
 * writing this data shouldn't be harmful even in those cases. */
1837
 * writing this data shouldn't be harmful even in those cases. */
1320
static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv)
1838
static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
1321
{
-
 
1322
#define GEN8_PPAT_UC		(0<<0)
-
 
1323
#define GEN8_PPAT_WC		(1<<0)
-
 
1324
#define GEN8_PPAT_WT		(2<<0)
-
 
1325
#define GEN8_PPAT_WB		(3<<0)
-
 
1326
#define GEN8_PPAT_ELLC_OVERRIDE	(0<<2)
-
 
1327
/* FIXME(BDW): Bspec is completely confused about cache control bits. */
-
 
1328
#define GEN8_PPAT_LLC		(1<<2)
-
 
1329
#define GEN8_PPAT_LLCELLC	(2<<2)
-
 
1330
#define GEN8_PPAT_LLCeLLC	(3<<2)
-
 
1331
#define GEN8_PPAT_AGE(x)	(x<<4)
-
 
1332
#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8))
1839
{
Line 1333... Line 1840...
1333
	uint64_t pat;
1840
	uint64_t pat;
1334
 
1841
 
1335
	pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
1842
	pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
Line 1345... Line 1852...
1345
	 * write would work. */
1852
	 * write would work. */
1346
	I915_WRITE(GEN8_PRIVATE_PAT, pat);
1853
	I915_WRITE(GEN8_PRIVATE_PAT, pat);
1347
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1854
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1348
}
1855
}
Line -... Line 1856...
-
 
1856
 
-
 
1857
static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
-
 
1858
{
-
 
1859
	uint64_t pat;
-
 
1860
 
-
 
1861
	/*
-
 
1862
	 * Map WB on BDW to snooped on CHV.
-
 
1863
	 *
-
 
1864
	 * Only the snoop bit has meaning for CHV, the rest is
-
 
1865
	 * ignored.
-
 
1866
	 *
-
 
1867
	 * Note that the harware enforces snooping for all page
-
 
1868
	 * table accesses. The snoop bit is actually ignored for
-
 
1869
	 * PDEs.
-
 
1870
	 */
-
 
1871
	pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
-
 
1872
	      GEN8_PPAT(1, 0) |
-
 
1873
	      GEN8_PPAT(2, 0) |
-
 
1874
	      GEN8_PPAT(3, 0) |
-
 
1875
	      GEN8_PPAT(4, CHV_PPAT_SNOOP) |
-
 
1876
	      GEN8_PPAT(5, CHV_PPAT_SNOOP) |
-
 
1877
	      GEN8_PPAT(6, CHV_PPAT_SNOOP) |
-
 
1878
	      GEN8_PPAT(7, CHV_PPAT_SNOOP);
-
 
1879
 
-
 
1880
	I915_WRITE(GEN8_PRIVATE_PAT, pat);
-
 
1881
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
-
 
1882
}
1349
 
1883
 
1350
static int gen8_gmch_probe(struct drm_device *dev,
1884
static int gen8_gmch_probe(struct drm_device *dev,
1351
			   size_t *gtt_total,
1885
			   size_t *gtt_total,
1352
			   size_t *stolen,
1886
			   size_t *stolen,
1353
			   phys_addr_t *mappable_base,
1887
			   phys_addr_t *mappable_base,
Line 1365... Line 1899...
1365
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
1899
	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
1366
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
1900
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
Line 1367... Line 1901...
1367
 
1901
 
Line -... Line 1902...
-
 
1902
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
-
 
1903
 
-
 
1904
	if (IS_CHERRYVIEW(dev)) {
-
 
1905
		*stolen = chv_get_stolen_size(snb_gmch_ctl);
1368
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1906
		gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
1369
 
-
 
1370
	*stolen = gen8_get_stolen_size(snb_gmch_ctl);
1907
	} else {
-
 
1908
	*stolen = gen8_get_stolen_size(snb_gmch_ctl);
-
 
1909
		gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
1371
 
1910
	}
Line -... Line 1911...
-
 
1911
 
1372
	gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
1912
	*gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
-
 
1913
 
-
 
1914
	if (IS_CHERRYVIEW(dev))
Line 1373... Line 1915...
1373
	*gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
1915
		chv_setup_private_ppat(dev_priv);
Line 1374... Line 1916...
1374
 
1916
	else
1375
	gen8_setup_private_ppat(dev_priv);
1917
		bdw_setup_private_ppat(dev_priv);
Line 1424... Line 1966...
1424
 
1966
 
1425
static void gen6_gmch_remove(struct i915_address_space *vm)
1967
static void gen6_gmch_remove(struct i915_address_space *vm)
Line 1426... Line 1968...
1426
{
1968
{
-
 
1969
 
-
 
1970
	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
-
 
1971
 
-
 
1972
	if (drm_mm_initialized(&vm->mm)) {
-
 
1973
		drm_mm_takedown(&vm->mm);
1427
 
1974
		list_del(&vm->global_link);
1428
	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
1975
	}
1429
	iounmap(gtt->gsm);
1976
	iounmap(gtt->gsm);
Line 1430... Line 1977...
1430
	teardown_scratch_page(vm->dev);
1977
	teardown_scratch_page(vm->dev);
Line 1447... Line 1994...
1447
 
1994
 
Line 1448... Line 1995...
1448
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
1995
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
1449
 
1996
 
1450
	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
-
 
Line 1451... Line 1997...
1451
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
1997
	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
1452
	dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
1998
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
Line 1453... Line 1999...
1453
 
1999
 
Line 1499... Line 2045...
1499
	/* GMADR is the PCI mmio aperture into the global GTT. */
2045
	/* GMADR is the PCI mmio aperture into the global GTT. */
1500
	DRM_INFO("Memory usable by graphics device = %zdM\n",
2046
	DRM_INFO("Memory usable by graphics device = %zdM\n",
1501
		 gtt->base.total >> 20);
2047
		 gtt->base.total >> 20);
1502
	DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2048
	DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
1503
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
2049
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
-
 
2050
#ifdef CONFIG_INTEL_IOMMU
-
 
2051
	if (intel_iommu_gfx_mapped)
-
 
2052
		DRM_INFO("VT-d active for gfx access\n");
-
 
2053
#endif
-
 
2054
	/*
-
 
2055
	 * i915.enable_ppgtt is read-only, so do an early pass to validate the
-
 
2056
	 * user's requested state against the hardware/driver capabilities.  We
-
 
2057
	 * do this now so that we can print out any log messages once rather
-
 
2058
	 * than every time we check intel_enable_ppgtt().
-
 
2059
	 */
-
 
2060
	i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
-
 
2061
	DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
Line 1504... Line 2062...
1504
 
2062
 
1505
	return 0;
2063
	return 0;
Line -... Line 2064...
-
 
2064
}
-
 
2065
 
-
 
2066
static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
-
 
2067
					      struct i915_address_space *vm)
-
 
2068
{
-
 
2069
	struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
-
 
2070
	if (vma == NULL)
-
 
2071
		return ERR_PTR(-ENOMEM);
-
 
2072
 
-
 
2073
	INIT_LIST_HEAD(&vma->vma_link);
-
 
2074
	INIT_LIST_HEAD(&vma->mm_list);
-
 
2075
	INIT_LIST_HEAD(&vma->exec_list);
-
 
2076
	vma->vm = vm;
-
 
2077
	vma->obj = obj;
-
 
2078
 
-
 
2079
	switch (INTEL_INFO(vm->dev)->gen) {
-
 
2080
	case 8:
-
 
2081
	case 7:
-
 
2082
	case 6:
-
 
2083
		if (i915_is_ggtt(vm)) {
-
 
2084
			vma->unbind_vma = ggtt_unbind_vma;
-
 
2085
			vma->bind_vma = ggtt_bind_vma;
-
 
2086
		} else {
-
 
2087
			vma->unbind_vma = ppgtt_unbind_vma;
-
 
2088
			vma->bind_vma = ppgtt_bind_vma;
-
 
2089
		}
-
 
2090
		break;
-
 
2091
	case 5:
-
 
2092
	case 4:
-
 
2093
	case 3:
-
 
2094
	case 2:
-
 
2095
		BUG_ON(!i915_is_ggtt(vm));
-
 
2096
		vma->unbind_vma = i915_ggtt_unbind_vma;
-
 
2097
		vma->bind_vma = i915_ggtt_bind_vma;
-
 
2098
		break;
-
 
2099
	default:
-
 
2100
		BUG();
-
 
2101
	}
-
 
2102
 
-
 
2103
	/* Keep GGTT vmas first to make debug easier */
-
 
2104
	if (i915_is_ggtt(vm))
-
 
2105
		list_add(&vma->vma_link, &obj->vma_list);
-
 
2106
	else
-
 
2107
		list_add_tail(&vma->vma_link, &obj->vma_list);
-
 
2108
 
-
 
2109
	return vma;
-
 
2110
}
-
 
2111
 
-
 
2112
struct i915_vma *
-
 
2113
i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
-
 
2114
				  struct i915_address_space *vm)
-
 
2115
{
-
 
2116
	struct i915_vma *vma;
-
 
2117
 
-
 
2118
	vma = i915_gem_obj_to_vma(obj, vm);
-
 
2119
	if (!vma)
-
 
2120
		vma = __i915_gem_vma_create(obj, vm);
-
 
2121
 
-
 
2122
	return vma;
1506
}
2123
}
1507
 
2124
 
1508
struct scatterlist *sg_next(struct scatterlist *sg)
2125
struct scatterlist *sg_next(struct scatterlist *sg)
1509
{
2126
{
Line 1517... Line 2134...
1517
    return sg;
2134
    return sg;
1518
}
2135
}
Line 1519... Line 2136...
1519
 
2136
 
1520
 
2137
 
1521
void __sg_free_table(struct sg_table *table, unsigned int max_ents,
2138
void __sg_free_table(struct sg_table *table, unsigned int max_ents,
1522
                     sg_free_fn *free_fn)
2139
             bool skip_first_chunk, sg_free_fn *free_fn)
Line 1523... Line 2140...
1523
{
2140
{
1524
    struct scatterlist *sgl, *next;
2141
    struct scatterlist *sgl, *next;
Line 1545... Line 2162...
1545
                sg_size = alloc_size;
2162
            sg_size = alloc_size;
1546
                next = NULL;
2163
            next = NULL;
1547
        }
2164
        }
Line 1548... Line 2165...
1548
 
2165
 
-
 
2166
        table->orig_nents -= sg_size;
1549
        table->orig_nents -= sg_size;
2167
        if (!skip_first_chunk) {
-
 
2168
            kfree(sgl);
-
 
2169
            skip_first_chunk = false;
1550
        kfree(sgl);
2170
        }
1551
        sgl = next;
2171
        sgl = next;
Line 1552... Line 2172...
1552
    }
2172
    }
1553
 
2173
 
1554
    table->sgl = NULL;
-
 
1555
}
2174
    table->sgl = NULL;
1556
 
2175
}
1557
void sg_free_table(struct sg_table *table)
2176
void sg_free_table(struct sg_table *table)
1558
{
2177
{
Line 1559... Line 2178...
1559
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
2178
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, NULL);
1560
}
2179
}
1561
 
2180
 
Line 1620... Line 2239...
1620
    } while (left);
2239
    } while (left);
Line 1621... Line 2240...
1621
 
2240
 
Line 1622... Line 2241...
1622
    return 0;
2241
    return 0;
1623
 
2242
 
Line 1624... Line 2243...
1624
err:
2243
err:
1625
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL);
2244
    __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, NULL);