Rev 4569 | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 4569 | Rev 6296 | ||
---|---|---|---|
Line 1... | Line 1... | ||
1 | /************************************************************************** |
1 | /************************************************************************** |
2 | * |
2 | * |
3 | * Copyright © 2009 VMware, Inc., Palo Alto, CA., USA |
3 | * Copyright © 2009-2015 VMware, Inc., Palo Alto, CA., USA |
4 | * All Rights Reserved. |
4 | * All Rights Reserved. |
5 | * |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * Permission is hereby granted, free of charge, to any person obtaining a |
7 | * copy of this software and associated documentation files (the |
7 | * copy of this software and associated documentation files (the |
8 | * "Software"), to deal in the Software without restriction, including |
8 | * "Software"), to deal in the Software without restriction, including |
Line 28... | Line 28... | ||
28 | #include "vmwgfx_drv.h" |
28 | #include "vmwgfx_drv.h" |
29 | #include |
29 | #include |
30 | #include |
30 | #include |
31 | #include |
31 | #include |
Line 32... | Line 32... | ||
32 | 32 | ||
- | 33 | static struct ttm_place vram_placement_flags = { |
|
- | 34 | .fpfn = 0, |
|
33 | static uint32_t vram_placement_flags = TTM_PL_FLAG_VRAM | |
35 | .lpfn = 0, |
- | 36 | .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED |
|
Line 34... | Line 37... | ||
34 | TTM_PL_FLAG_CACHED; |
37 | }; |
- | 38 | ||
35 | 39 | static struct ttm_place vram_ne_placement_flags = { |
|
36 | static uint32_t vram_ne_placement_flags = TTM_PL_FLAG_VRAM | |
40 | .fpfn = 0, |
- | 41 | .lpfn = 0, |
|
Line 37... | Line 42... | ||
37 | TTM_PL_FLAG_CACHED | |
42 | .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT |
- | 43 | }; |
|
- | 44 | ||
38 | TTM_PL_FLAG_NO_EVICT; |
45 | static struct ttm_place sys_placement_flags = { |
- | 46 | .fpfn = 0, |
|
Line 39... | Line 47... | ||
39 | 47 | .lpfn = 0, |
|
40 | static uint32_t sys_placement_flags = TTM_PL_FLAG_SYSTEM | |
48 | .flags = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED |
- | 49 | }; |
|
41 | TTM_PL_FLAG_CACHED; |
50 | |
- | 51 | static struct ttm_place sys_ne_placement_flags = { |
|
Line 42... | Line 52... | ||
42 | 52 | .fpfn = 0, |
|
- | 53 | .lpfn = 0, |
|
- | 54 | .flags = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT |
|
43 | static uint32_t sys_ne_placement_flags = TTM_PL_FLAG_SYSTEM | |
55 | }; |
- | 56 | ||
Line 44... | Line 57... | ||
44 | TTM_PL_FLAG_CACHED | |
57 | static struct ttm_place gmr_placement_flags = { |
45 | TTM_PL_FLAG_NO_EVICT; |
58 | .fpfn = 0, |
- | 59 | .lpfn = 0, |
|
46 | 60 | .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED |
|
- | 61 | }; |
|
Line 47... | Line 62... | ||
47 | static uint32_t gmr_placement_flags = VMW_PL_FLAG_GMR | |
62 | |
- | 63 | static struct ttm_place gmr_ne_placement_flags = { |
|
- | 64 | .fpfn = 0, |
|
48 | TTM_PL_FLAG_CACHED; |
65 | .lpfn = 0, |
- | 66 | .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT |
|
Line 49... | Line 67... | ||
49 | 67 | }; |
|
50 | static uint32_t gmr_ne_placement_flags = VMW_PL_FLAG_GMR | |
68 | |
51 | TTM_PL_FLAG_CACHED | |
69 | static struct ttm_place mob_placement_flags = { |
- | 70 | .fpfn = 0, |
|
- | 71 | .lpfn = 0, |
|
- | 72 | .flags = VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED |
|
- | 73 | }; |
|
52 | TTM_PL_FLAG_NO_EVICT; |
74 | |
53 | 75 | static struct ttm_place mob_ne_placement_flags = { |
|
54 | static uint32_t mob_placement_flags = VMW_PL_FLAG_MOB | |
76 | .fpfn = 0, |
55 | TTM_PL_FLAG_CACHED; |
77 | .lpfn = 0, |
56 | 78 | .flags = VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT |
|
Line 57... | Line 79... | ||
57 | struct ttm_placement vmw_vram_placement = { |
79 | }; |
- | 80 | ||
- | 81 | struct ttm_placement vmw_vram_placement = { |
|
- | 82 | .num_placement = 1, |
|
58 | .fpfn = 0, |
83 | .placement = &vram_placement_flags, |
- | 84 | .num_busy_placement = 1, |
|
- | 85 | .busy_placement = &vram_placement_flags |
|
- | 86 | }; |
|
59 | .lpfn = 0, |
87 | |
- | 88 | static struct ttm_place vram_gmr_placement_flags[] = { |
|
60 | .num_placement = 1, |
89 | { |
Line 61... | Line 90... | ||
61 | .placement = &vram_placement_flags, |
90 | .fpfn = 0, |
- | 91 | .lpfn = 0, |
|
- | 92 | .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED |
|
- | 93 | }, { |
|
62 | .num_busy_placement = 1, |
94 | .fpfn = 0, |
- | 95 | .lpfn = 0, |
|
- | 96 | .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED |
|
- | 97 | } |
|
63 | .busy_placement = &vram_placement_flags |
98 | }; |
- | 99 | ||
64 | }; |
100 | static struct ttm_place gmr_vram_placement_flags[] = { |
Line 65... | Line 101... | ||
65 | 101 | { |
|
66 | static uint32_t vram_gmr_placement_flags[] = { |
- | |
67 | TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED, |
- | |
68 | VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED |
102 | .fpfn = 0, |
69 | }; |
103 | .lpfn = 0, |
70 | 104 | .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED |
|
71 | static uint32_t gmr_vram_placement_flags[] = { |
105 | }, { |
72 | VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED, |
106 | .fpfn = 0, |
Line 73... | Line 107... | ||
73 | TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED |
107 | .lpfn = 0, |
- | 108 | .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED |
|
- | 109 | } |
|
- | 110 | }; |
|
74 | }; |
111 | |
- | 112 | struct ttm_placement vmw_vram_gmr_placement = { |
|
- | 113 | .num_placement = 2, |
|
- | 114 | .placement = vram_gmr_placement_flags, |
|
- | 115 | .num_busy_placement = 1, |
|
75 | 116 | .busy_placement = &gmr_placement_flags |
|
- | 117 | }; |
|
- | 118 | ||
76 | struct ttm_placement vmw_vram_gmr_placement = { |
119 | static struct ttm_place vram_gmr_ne_placement_flags[] = { |
Line 77... | Line 120... | ||
77 | .fpfn = 0, |
120 | { |
78 | .lpfn = 0, |
- | |
79 | .num_placement = 2, |
- | |
80 | .placement = vram_gmr_placement_flags, |
121 | .fpfn = 0, |
81 | .num_busy_placement = 1, |
122 | .lpfn = 0, |
82 | .busy_placement = &gmr_placement_flags |
123 | .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED | |
83 | }; |
124 | TTM_PL_FLAG_NO_EVICT |
84 | 125 | }, { |
|
Line 85... | Line 126... | ||
85 | static uint32_t vram_gmr_ne_placement_flags[] = { |
126 | .fpfn = 0, |
86 | TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT, |
- | |
87 | VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED | TTM_PL_FLAG_NO_EVICT |
- | |
88 | }; |
127 | .lpfn = 0, |
89 | 128 | .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED | |
|
90 | struct ttm_placement vmw_vram_gmr_ne_placement = { |
129 | TTM_PL_FLAG_NO_EVICT |
91 | .fpfn = 0, |
130 | } |
92 | .lpfn = 0, |
131 | }; |
Line 93... | Line 132... | ||
93 | .num_placement = 2, |
132 | |
94 | .placement = vram_gmr_ne_placement_flags, |
- | |
95 | .num_busy_placement = 1, |
- | |
96 | .busy_placement = &gmr_ne_placement_flags |
133 | struct ttm_placement vmw_vram_gmr_ne_placement = { |
97 | }; |
134 | .num_placement = 2, |
98 | 135 | .placement = vram_gmr_ne_placement_flags, |
|
99 | struct ttm_placement vmw_vram_sys_placement = { |
136 | .num_busy_placement = 1, |
100 | .fpfn = 0, |
137 | .busy_placement = &gmr_ne_placement_flags |
Line 101... | Line 138... | ||
101 | .lpfn = 0, |
138 | }; |
102 | .num_placement = 1, |
- | |
103 | .placement = &vram_placement_flags, |
- | |
104 | .num_busy_placement = 1, |
139 | |
105 | .busy_placement = &sys_placement_flags |
140 | struct ttm_placement vmw_vram_sys_placement = { |
106 | }; |
141 | .num_placement = 1, |
107 | 142 | .placement = &vram_placement_flags, |
|
108 | struct ttm_placement vmw_vram_ne_placement = { |
143 | .num_busy_placement = 1, |
Line 109... | Line 144... | ||
109 | .fpfn = 0, |
144 | .busy_placement = &sys_placement_flags |
110 | .lpfn = 0, |
- | |
111 | .num_placement = 1, |
- | |
112 | .placement = &vram_ne_placement_flags, |
145 | }; |
113 | .num_busy_placement = 1, |
146 | |
114 | .busy_placement = &vram_ne_placement_flags |
147 | struct ttm_placement vmw_vram_ne_placement = { |
115 | }; |
148 | .num_placement = 1, |
116 | 149 | .placement = &vram_ne_placement_flags, |
|
Line 117... | Line 150... | ||
117 | struct ttm_placement vmw_sys_placement = { |
150 | .num_busy_placement = 1, |
- | 151 | .busy_placement = &vram_ne_placement_flags |
|
- | 152 | }; |
|
- | 153 | ||
118 | .fpfn = 0, |
154 | struct ttm_placement vmw_sys_placement = { |
- | 155 | .num_placement = 1, |
|
- | 156 | .placement = &sys_placement_flags, |
|
- | 157 | .num_busy_placement = 1, |
|
119 | .lpfn = 0, |
158 | .busy_placement = &sys_placement_flags |
- | 159 | }; |
|
- | 160 | ||
- | 161 | struct ttm_placement vmw_sys_ne_placement = { |
|
120 | .num_placement = 1, |
162 | .num_placement = 1, |
- | 163 | .placement = &sys_ne_placement_flags, |
|
- | 164 | .num_busy_placement = 1, |
|
- | 165 | .busy_placement = &sys_ne_placement_flags |
|
121 | .placement = &sys_placement_flags, |
166 | }; |
- | 167 | ||
122 | .num_busy_placement = 1, |
168 | static struct ttm_place evictable_placement_flags[] = { |
Line 123... | Line 169... | ||
123 | .busy_placement = &sys_placement_flags |
169 | { |
124 | }; |
- | |
125 | - | ||
126 | struct ttm_placement vmw_sys_ne_placement = { |
170 | .fpfn = 0, |
127 | .fpfn = 0, |
171 | .lpfn = 0, |
128 | .lpfn = 0, |
172 | .flags = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED |
129 | .num_placement = 1, |
173 | }, { |
130 | .placement = &sys_ne_placement_flags, |
174 | .fpfn = 0, |
Line 131... | Line 175... | ||
131 | .num_busy_placement = 1, |
175 | .lpfn = 0, |
132 | .busy_placement = &sys_ne_placement_flags |
- | |
133 | }; |
- | |
134 | 176 | .flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED |
|
135 | static uint32_t evictable_placement_flags[] = { |
177 | }, { |
136 | TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED, |
178 | .fpfn = 0, |
137 | TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED, |
179 | .lpfn = 0, |
138 | VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED, |
180 | .flags = VMW_PL_FLAG_GMR | TTM_PL_FLAG_CACHED |
Line 139... | Line 181... | ||
139 | VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED |
181 | }, { |
140 | }; |
- | |
141 | - | ||
142 | struct ttm_placement vmw_evictable_placement = { |
182 | .fpfn = 0, |
143 | .fpfn = 0, |
183 | .lpfn = 0, |
144 | .lpfn = 0, |
184 | .flags = VMW_PL_FLAG_MOB | TTM_PL_FLAG_CACHED |
145 | .num_placement = 4, |
185 | } |
146 | .placement = evictable_placement_flags, |
186 | }; |
Line -... | Line 187... | ||
- | 187 | ||
- | 188 | struct ttm_placement vmw_evictable_placement = { |
|
- | 189 | .num_placement = 4, |
|
- | 190 | .placement = evictable_placement_flags, |
|
- | 191 | .num_busy_placement = 1, |
|
- | 192 | .busy_placement = &sys_placement_flags |
|
- | 193 | }; |
|
147 | .num_busy_placement = 1, |
194 | |
148 | .busy_placement = &sys_placement_flags |
195 | struct ttm_placement vmw_srf_placement = { |
149 | }; |
196 | .num_placement = 1, |
150 | 197 | .num_busy_placement = 2, |
|
151 | struct ttm_placement vmw_srf_placement = { |
198 | .placement = &gmr_placement_flags, |
Line 766... | Line 813... | ||
766 | { |
813 | { |
767 | return 0; |
814 | return 0; |
768 | } |
815 | } |
Line 769... | Line 816... | ||
769 | 816 | ||
770 | /** |
- | |
771 | * FIXME: We're using the old vmware polling method to sync. |
- | |
772 | * Do this with fences instead. |
- | |
773 | */ |
- | |
774 | - | ||
775 | static void *vmw_sync_obj_ref(void *sync_obj) |
- | |
776 | { |
- | |
777 | - | ||
778 | return (void *) |
- | |
779 | vmw_fence_obj_reference((struct vmw_fence_obj *) sync_obj); |
- | |
780 | } |
- | |
781 | - | ||
782 | static void vmw_sync_obj_unref(void **sync_obj) |
- | |
783 | { |
- | |
784 | vmw_fence_obj_unreference((struct vmw_fence_obj **) sync_obj); |
- | |
785 | } |
- | |
786 | - | ||
787 | static int vmw_sync_obj_flush(void *sync_obj) |
- | |
788 | { |
- | |
789 | vmw_fence_obj_flush((struct vmw_fence_obj *) sync_obj); |
- | |
790 | return 0; |
- | |
791 | } |
- | |
792 | - | ||
793 | static bool vmw_sync_obj_signaled(void *sync_obj) |
- | |
794 | { |
- | |
795 | return vmw_fence_obj_signaled((struct vmw_fence_obj *) sync_obj, |
- | |
796 | DRM_VMW_FENCE_FLAG_EXEC); |
- | |
797 | - | ||
798 | } |
- | |
799 | - | ||
800 | static int vmw_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) |
- | |
801 | { |
- | |
802 | return vmw_fence_obj_wait((struct vmw_fence_obj *) sync_obj, |
- | |
803 | DRM_VMW_FENCE_FLAG_EXEC, |
- | |
804 | lazy, interruptible, |
- | |
805 | VMW_FENCE_WAIT_TIMEOUT); |
- | |
806 | } |
- | |
807 | - | ||
808 | /** |
817 | /** |
809 | * vmw_move_notify - TTM move_notify_callback |
818 | * vmw_move_notify - TTM move_notify_callback |
810 | * |
819 | * |
811 | * @bo: The TTM buffer object about to move. |
820 | * @bo: The TTM buffer object about to move. |
812 | * @mem: The truct ttm_mem_reg indicating to what memory |
821 | * @mem: The struct ttm_mem_reg indicating to what memory |
813 | * region the move is taking place. |
822 | * region the move is taking place. |
814 | * |
823 | * |
815 | * Calls move_notify for all subsystems needing it. |
824 | * Calls move_notify for all subsystems needing it. |
816 | * (currently only resources). |
825 | * (currently only resources). |
Line 827... | Line 836... | ||
827 | * |
836 | * |
828 | * @bo: The TTM buffer object about to be swapped out. |
837 | * @bo: The TTM buffer object about to be swapped out. |
829 | */ |
838 | */ |
830 | static void vmw_swap_notify(struct ttm_buffer_object *bo) |
839 | static void vmw_swap_notify(struct ttm_buffer_object *bo) |
831 | { |
840 | { |
832 | struct ttm_bo_device *bdev = bo->bdev; |
- | |
833 | - | ||
834 | // spin_lock(&bdev->fence_lock); |
- | |
835 | // ttm_bo_wait(bo, false, false, false); |
841 | ttm_bo_wait(bo, false, false, false); |
836 | // spin_unlock(&bdev->fence_lock); |
- | |
837 | } |
842 | } |
Line 838... | Line 843... | ||
838 | 843 | ||
839 | 844 | ||
Line 844... | Line 849... | ||
844 | .invalidate_caches = vmw_invalidate_caches, |
849 | .invalidate_caches = vmw_invalidate_caches, |
845 | .init_mem_type = vmw_init_mem_type, |
850 | .init_mem_type = vmw_init_mem_type, |
846 | .evict_flags = vmw_evict_flags, |
851 | .evict_flags = vmw_evict_flags, |
847 | .move = NULL, |
852 | .move = NULL, |
848 | .verify_access = vmw_verify_access, |
853 | .verify_access = vmw_verify_access, |
849 | .sync_obj_signaled = vmw_sync_obj_signaled, |
- | |
850 | .sync_obj_wait = vmw_sync_obj_wait, |
- | |
851 | .sync_obj_flush = vmw_sync_obj_flush, |
- | |
852 | .sync_obj_unref = vmw_sync_obj_unref, |
- | |
853 | .sync_obj_ref = vmw_sync_obj_ref, |
- | |
854 | .move_notify = vmw_move_notify, |
854 | .move_notify = vmw_move_notify, |
855 | .swap_notify = vmw_swap_notify, |
855 | .swap_notify = vmw_swap_notify, |
856 | .fault_reserve_notify = &vmw_ttm_fault_reserve_notify, |
856 | .fault_reserve_notify = &vmw_ttm_fault_reserve_notify, |
857 | .io_mem_reserve = &vmw_ttm_io_mem_reserve, |
857 | .io_mem_reserve = &vmw_ttm_io_mem_reserve, |
858 | .io_mem_free = &vmw_ttm_io_mem_free, |
858 | .io_mem_free = &vmw_ttm_io_mem_free, |
859 | }; |
859 | };><>><>><>> |
860 | - | ||
861 | - | ||
862 | struct scatterlist *sg_next(struct scatterlist *sg) |
- | |
863 | { |
- | |
864 | if (sg_is_last(sg)) |
- | |
865 | return NULL; |
- | |
866 | - | ||
867 | sg++; |
- | |
868 | if (unlikely(sg_is_chain(sg))) |
- | |
869 | sg = sg_chain_ptr(sg); |
- | |
870 | - | ||
871 | return sg; |
- | |
872 | } |
- | |
873 | - | ||
874 | - | ||
875 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
- | |
876 | sg_free_fn *free_fn) |
- | |
877 | { |
- | |
878 | struct scatterlist *sgl, *next; |
- | |
879 | - | ||
880 | if (unlikely(!table->sgl)) |
- | |
881 | return; |
- | |
882 | - | ||
883 | sgl = table->sgl; |
- | |
884 | while (table->orig_nents) { |
- | |
885 | unsigned int alloc_size = table->orig_nents; |
- | |
886 | unsigned int sg_size; |
- | |
887 | - | ||
888 | /* |
- | |
889 | * If we have more than max_ents segments left, |
- | |
890 | * then assign 'next' to the sg table after the current one. |
- | |
891 | * sg_size is then one less than alloc size, since the last |
- | |
892 | * element is the chain pointer. |
- | |
893 | */ |
- | |
894 | if (alloc_size > max_ents) { |
- | |
895 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
- | |
896 | alloc_size = max_ents; |
- | |
897 | sg_size = alloc_size - 1; |
- | |
898 | } else { |
- | |
899 | sg_size = alloc_size; |
- | |
900 | next = NULL; |
- | |
901 | } |
- | |
902 | - | ||
903 | table->orig_nents -= sg_size; |
- | |
904 | kfree(sgl); |
- | |
905 | sgl = next; |
- | |
906 | } |
- | |
907 | - | ||
908 | table->sgl = NULL; |
- | |
909 | } |
- | |
910 | - | ||
911 | void sg_free_table(struct sg_table *table) |
- | |
912 | { |
- | |
913 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
- | |
914 | } |
- | |
915 | - | ||
916 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
- | |
917 | { |
- | |
918 | struct scatterlist *sg, *prv; |
- | |
919 | unsigned int left; |
- | |
920 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
- | |
921 | - | ||
922 | #ifndef ARCH_HAS_SG_CHAIN |
- | |
923 | BUG_ON(nents > max_ents); |
- | |
924 | #endif |
- | |
925 | - | ||
926 | memset(table, 0, sizeof(*table)); |
- | |
927 | - | ||
928 | left = nents; |
- | |
929 | prv = NULL; |
- | |
930 | do { |
- | |
931 | unsigned int sg_size, alloc_size = left; |
- | |
932 | - | ||
933 | if (alloc_size > max_ents) { |
- | |
934 | alloc_size = max_ents; |
- | |
935 | sg_size = alloc_size - 1; |
- | |
936 | } else |
- | |
937 | sg_size = alloc_size; |
- | |
938 | - | ||
939 | left -= sg_size; |
- | |
940 | - | ||
941 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
- | |
942 | if (unlikely(!sg)) { |
- | |
943 | /* |
- | |
944 | * Adjust entry count to reflect that the last |
- | |
945 | * entry of the previous table won't be used for |
- | |
946 | * linkage. Without this, sg_kfree() may get |
- | |
947 | * confused. |
- | |
948 | */ |
- | |
949 | if (prv) |
- | |
950 | table->nents = ++table->orig_nents; |
- | |
951 | - | ||
952 | goto err; |
- | |
953 | } |
- | |
954 | - | ||
955 | sg_init_table(sg, alloc_size); |
- | |
956 | table->nents = table->orig_nents += sg_size; |
- | |
957 | - | ||
958 | /* |
- | |
959 | * If this is the first mapping, assign the sg table header. |
- | |
960 | * If this is not the first mapping, chain previous part. |
- | |
961 | */ |
- | |
962 | if (prv) |
- | |
963 | sg_chain(prv, max_ents, sg); |
- | |
964 | else |
- | |
965 | table->sgl = sg; |
- | |
966 | - | ||
967 | /* |
- | |
968 | * If no more entries after this one, mark the end |
- | |
969 | */ |
- | |
970 | if (!left) |
- | |
971 | sg_mark_end(&sg[sg_size - 1]); |
- | |
972 | - | ||
973 | prv = sg; |
- | |
974 | } while (left); |
- | |
975 | - | ||
976 | return 0; |
- | |
977 | - | ||
978 | err: |
- | |
979 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
- | |
980 | - | ||
981 | return -ENOMEM; |
- | |
982 | } |
- | |
983 | - | ||
984 | - | ||
985 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
- | |
986 | { |
- | |
987 | memset(sgl, 0, sizeof(*sgl) * nents); |
- | |
988 | #ifdef CONFIG_DEBUG_SG |
- | |
989 | { |
- | |
990 | unsigned int i; |
- | |
991 | for (i = 0; i < nents; i++) |
- | |
992 | sgl[i].sg_magic = SG_MAGIC; |
- | |
993 | } |
- | |
994 | #endif |
- | |
995 | sg_mark_end(&sgl[nents - 1]); |
- | |
996 | } |
- | |
997 | - | ||
998 | - | ||
999 | void __sg_page_iter_start(struct sg_page_iter *piter, |
- | |
1000 | struct scatterlist *sglist, unsigned int nents, |
- | |
1001 | unsigned long pgoffset) |
- | |
1002 | { |
- | |
1003 | piter->__pg_advance = 0; |
- | |
1004 | piter->__nents = nents; |
- | |
1005 | - | ||
1006 | piter->sg = sglist; |
- | |
1007 | piter->sg_pgoffset = pgoffset; |
- | |
1008 | } |
- | |
1009 | - | ||
1010 | static int sg_page_count(struct scatterlist *sg) |
- | |
1011 | { |
- | |
1012 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
- | |
1013 | } |
- | |
1014 | - | ||
1015 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
- | |
1016 | { |
- | |
1017 | if (!piter->__nents || !piter->sg) |
- | |
1018 | return false; |
- | |
1019 | - | ||
1020 | piter->sg_pgoffset += piter->__pg_advance; |
- | |
1021 | piter->__pg_advance = 1; |
- | |
1022 | - | ||
1023 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
- | |
1024 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
- | |
1025 | piter->sg = sg_next(piter->sg); |
- | |
1026 | if (!--piter->__nents || !piter->sg) |
- | |
1027 | return false; |
- | |
1028 | } |
- | |
1029 | - | ||
1030 | return true; |
- | |
1031 | } |
- | |
1032 | EXPORT_SYMBOL(__sg_page_iter_next); |
- | |
1033 | - | ||
1034 | - | ||
1035 | int sg_alloc_table_from_pages(struct sg_table *sgt, |
- | |
1036 | struct page **pages, unsigned int n_pages, |
- | |
1037 | unsigned long offset, unsigned long size, |
- | |
1038 | gfp_t gfp_mask) |
- | |
1039 | { |
- | |
1040 | unsigned int chunks; |
- | |
1041 | unsigned int i; |
- | |
1042 | unsigned int cur_page; |
- | |
1043 | int ret; |
- | |
1044 | struct scatterlist *s; |
- | |
1045 | - | ||
1046 | /* compute number of contiguous chunks */ |
- | |
1047 | chunks = 1; |
- | |
1048 | for (i = 1; i < n_pages; ++i) |
- | |
1049 | if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) |
- | |
1050 | ++chunks; |
- | |
1051 | - | ||
1052 | ret = sg_alloc_table(sgt, chunks, gfp_mask); |
- | |
1053 | if (unlikely(ret)) |
- | |
1054 | return ret; |
- | |
1055 | - | ||
1056 | /* merging chunks and putting them into the scatterlist */ |
- | |
1057 | cur_page = 0; |
- | |
1058 | for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { |
- | |
1059 | unsigned long chunk_size; |
- | |
1060 | unsigned int j; |
- | |
1061 | - | ||
1062 | /* look for the end of the current chunk */ |
- | |
1063 | for (j = cur_page + 1; j < n_pages; ++j) |
- | |
1064 | if (page_to_pfn(pages[j]) != |
- | |
1065 | page_to_pfn(pages[j - 1]) + 1) |
- | |
1066 | break; |
- | |
1067 | - | ||
1068 | chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset; |
- | |
1069 | sg_set_page(s, pages[cur_page], min(size, chunk_size), offset); |
- | |
1070 | size -= chunk_size; |
- | |
1071 | offset = 0; |
- | |
1072 | cur_page = j; |
- | |
1073 | } |
- | |
1074 | - | ||
1075 | return 0; |
- | |
1076 | } |
- | |
1077 | - | ||
1078 | int dma_map_sg(struct device *dev, struct scatterlist *sglist, |
- | |
1079 | int nelems, int dir) |
- | |
1080 | { |
- | |
1081 | struct scatterlist *s; |
- | |
1082 | int i; |
- | |
1083 | - | ||
1084 | for_each_sg(sglist, s, nelems, i) { |
- | |
1085 | s->dma_address = (dma_addr_t)sg_phys(s); |
- | |
1086 | #ifdef CONFIG_NEED_SG_DMA_LENGTH |
- | |
1087 | s->dma_length = s->length; |
- | |
1088 | #endif |
- | |
1089 | } |
- | |
1090 | - | ||
1091 | return nelems; |
- | |
1092 | }><>>>>><>><>><>> |
- | |
1093 | - |