Eric Lee / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 1d540db76d9028af90d181f509eeb918a7cb6e41

Authored by Xianzhong
1 parent 22adb879b2
Exists in smarc_8mm_imx_4.14.98_2.0.0_ga and in 4 other branches smarc-imx6_4.14.98_2.0.0_ga, smarc-imx7_4.14.98_2.0.0_ga, smarc_8m_00d0_imx_4.14.98_2.0.0_ga, smarc_8m_imx_4.14.98_2.0.0_ga

MGS-4645-2 [#imx-1373] fix command object race condition risk 

Vulkan driver has problem to commit command without device mutex,
command object has race condition risk without mutex protection.

move _ProcessUserCommandBufferList into command mutex region.

Signed-off-by: Xianzhong <xianzhong.li@nxp.com>
(cherry picked from commit 10120a64fd9342c8effd21eb62bc51f88f1493cb)

Showing 1 changed file with 8 additions and 8 deletions Inline Diff

drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_command.c
Diff comments   View file @ 1d540db
1 /**************************************************************************** 1 /****************************************************************************
2 * 2 *
3 * The MIT License (MIT) 3 * The MIT License (MIT)
4 * 4 *
5 * Copyright (c) 2014 - 2018 Vivante Corporation 5 * Copyright (c) 2014 - 2018 Vivante Corporation
6 * 6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a 7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"), 8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation 9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the 11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions: 12 * Software is furnished to do so, subject to the following conditions:
13 * 13 *
14 * The above copyright notice and this permission notice shall be included in 14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software. 15 * all copies or substantial portions of the Software.
16 * 16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 * DEALINGS IN THE SOFTWARE. 23 * DEALINGS IN THE SOFTWARE.
24 * 24 *
25 ***************************************************************************** 25 *****************************************************************************
26 * 26 *
27 * The GPL License (GPL) 27 * The GPL License (GPL)
28 * 28 *
29 * Copyright (C) 2014 - 2018 Vivante Corporation 29 * Copyright (C) 2014 - 2018 Vivante Corporation
30 * 30 *
31 * This program is free software; you can redistribute it and/or 31 * This program is free software; you can redistribute it and/or
32 * modify it under the terms of the GNU General Public License 32 * modify it under the terms of the GNU General Public License
33 * as published by the Free Software Foundation; either version 2 33 * as published by the Free Software Foundation; either version 2
34 * of the License, or (at your option) any later version. 34 * of the License, or (at your option) any later version.
35 * 35 *
36 * This program is distributed in the hope that it will be useful, 36 * This program is distributed in the hope that it will be useful,
37 * but WITHOUT ANY WARRANTY; without even the implied warranty of 37 * but WITHOUT ANY WARRANTY; without even the implied warranty of
38 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 38 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
39 * GNU General Public License for more details. 39 * GNU General Public License for more details.
40 * 40 *
41 * You should have received a copy of the GNU General Public License 41 * You should have received a copy of the GNU General Public License
42 * along with this program; if not, write to the Free Software Foundation, 42 * along with this program; if not, write to the Free Software Foundation,
43 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 43 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
44 * 44 *
45 ***************************************************************************** 45 *****************************************************************************
46 * 46 *
47 * Note: This software is released under dual MIT and GPL licenses. A 47 * Note: This software is released under dual MIT and GPL licenses. A
48 * recipient may use this file under the terms of either the MIT license or 48 * recipient may use this file under the terms of either the MIT license or
49 * GPL License. If you wish to use only one license not the other, you can 49 * GPL License. If you wish to use only one license not the other, you can
50 * indicate your decision by deleting one of the above license notices in your 50 * indicate your decision by deleting one of the above license notices in your
51 * version of this file. 51 * version of this file.
52 * 52 *
53 *****************************************************************************/ 53 *****************************************************************************/
54 54
55 55
56 #include "gc_hal_kernel_precomp.h" 56 #include "gc_hal_kernel_precomp.h"
57 #include "gc_hal_kernel_context.h" 57 #include "gc_hal_kernel_context.h"
58 58
59 #define _GC_OBJ_ZONE gcvZONE_COMMAND 59 #define _GC_OBJ_ZONE gcvZONE_COMMAND
60 60
61 /******************************************************************************\ 61 /******************************************************************************\
62 ********************************* Support Code ********************************* 62 ********************************* Support Code *********************************
63 \******************************************************************************/ 63 \******************************************************************************/
64 64
65 /******************************************************************************* 65 /*******************************************************************************
66 ** 66 **
67 ** _NewQueue 67 ** _NewQueue
68 ** 68 **
69 ** Allocate a new command queue. 69 ** Allocate a new command queue.
70 ** 70 **
71 ** INPUT: 71 ** INPUT:
72 ** 72 **
73 ** gckCOMMAND Command 73 ** gckCOMMAND Command
74 ** Pointer to an gckCOMMAND object. 74 ** Pointer to an gckCOMMAND object.
75 ** 75 **
76 ** gctBOOL Stalled 76 ** gctBOOL Stalled
77 ** Indicate if hardware is stalled already. 77 ** Indicate if hardware is stalled already.
78 ** 78 **
79 ** OUTPUT: 79 ** OUTPUT:
80 ** 80 **
81 ** gckCOMMAND Command 81 ** gckCOMMAND Command
82 ** gckCOMMAND object has been updated with a new command queue. 82 ** gckCOMMAND object has been updated with a new command queue.
83 */ 83 */
84 gceSTATUS 84 gceSTATUS
85 _NewQueue( 85 _NewQueue(
86 IN OUT gckCOMMAND Command, 86 IN OUT gckCOMMAND Command,
87 IN gctBOOL Stalled 87 IN gctBOOL Stalled
88 ) 88 )
89 { 89 {
90 gceSTATUS status; 90 gceSTATUS status;
91 gctINT currentIndex, newIndex; 91 gctINT currentIndex, newIndex;
92 gctPHYS_ADDR_T physical; 92 gctPHYS_ADDR_T physical;
93 93
94 gcmkHEADER_ARG("Command=0x%x", Command); 94 gcmkHEADER_ARG("Command=0x%x", Command);
95 95
96 /* Switch to the next command buffer. */ 96 /* Switch to the next command buffer. */
97 currentIndex = Command->index; 97 currentIndex = Command->index;
98 newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES; 98 newIndex = (currentIndex + 1) % gcdCOMMAND_QUEUES;
99 99
100 /* Wait for availability. */ 100 /* Wait for availability. */
101 #if gcdDUMP_COMMAND 101 #if gcdDUMP_COMMAND
102 gcmkPRINT("@[kernel.waitsignal]"); 102 gcmkPRINT("@[kernel.waitsignal]");
103 #endif 103 #endif
104 104
105 gcmkONERROR(gckOS_WaitSignal( 105 gcmkONERROR(gckOS_WaitSignal(
106 Command->os, 106 Command->os,
107 Command->queues[newIndex].signal, 107 Command->queues[newIndex].signal,
108 gcvFALSE, 108 gcvFALSE,
109 gcvINFINITE 109 gcvINFINITE
110 )); 110 ));
111 111
112 #if gcmIS_DEBUG(gcdDEBUG_TRACE) 112 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
113 if (newIndex < currentIndex) 113 if (newIndex < currentIndex)
114 { 114 {
115 Command->wrapCount += 1; 115 Command->wrapCount += 1;
116 116
117 gcmkTRACE_ZONE_N( 117 gcmkTRACE_ZONE_N(
118 gcvLEVEL_INFO, gcvZONE_COMMAND, 118 gcvLEVEL_INFO, gcvZONE_COMMAND,
119 2 * 4, 119 2 * 4,
120 "%s(%d): queue array wrapped around.\n", 120 "%s(%d): queue array wrapped around.\n",
121 __FUNCTION__, __LINE__ 121 __FUNCTION__, __LINE__
122 ); 122 );
123 } 123 }
124 124
125 gcmkTRACE_ZONE_N( 125 gcmkTRACE_ZONE_N(
126 gcvLEVEL_INFO, gcvZONE_COMMAND, 126 gcvLEVEL_INFO, gcvZONE_COMMAND,
127 3 * 4, 127 3 * 4,
128 "%s(%d): total queue wrap arounds %d.\n", 128 "%s(%d): total queue wrap arounds %d.\n",
129 __FUNCTION__, __LINE__, Command->wrapCount 129 __FUNCTION__, __LINE__, Command->wrapCount
130 ); 130 );
131 131
132 gcmkTRACE_ZONE_N( 132 gcmkTRACE_ZONE_N(
133 gcvLEVEL_INFO, gcvZONE_COMMAND, 133 gcvLEVEL_INFO, gcvZONE_COMMAND,
134 3 * 4, 134 3 * 4,
135 "%s(%d): switched to queue %d.\n", 135 "%s(%d): switched to queue %d.\n",
136 __FUNCTION__, __LINE__, newIndex 136 __FUNCTION__, __LINE__, newIndex
137 ); 137 );
138 #endif 138 #endif
139 139
140 /* Update gckCOMMAND object with new command queue. */ 140 /* Update gckCOMMAND object with new command queue. */
141 Command->index = newIndex; 141 Command->index = newIndex;
142 Command->newQueue = gcvTRUE; 142 Command->newQueue = gcvTRUE;
143 #if USE_KERNEL_VIRTUAL_BUFFERS 143 #if USE_KERNEL_VIRTUAL_BUFFERS
144 if (Command->kernel->virtualCommandBuffer) 144 if (Command->kernel->virtualCommandBuffer)
145 { 145 {
146 gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = gcvNULL; 146 gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = gcvNULL;
147 147
148 Command->virtualMemory = Command->queues[newIndex].physical; 148 Command->virtualMemory = Command->queues[newIndex].physical;
149 149
150 commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) Command->virtualMemory; 150 commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) Command->virtualMemory;
151 151
152 Command->physHandle = commandBuffer->virtualBuffer.physical; 152 Command->physHandle = commandBuffer->virtualBuffer.physical;
153 } 153 }
154 else 154 else
155 #endif 155 #endif
156 { 156 {
157 Command->physHandle = Command->queues[newIndex].physical; 157 Command->physHandle = Command->queues[newIndex].physical;
158 } 158 }
159 159
160 Command->logical = Command->queues[newIndex].logical; 160 Command->logical = Command->queues[newIndex].logical;
161 Command->address = Command->queues[newIndex].address; 161 Command->address = Command->queues[newIndex].address;
162 Command->offset = 0; 162 Command->offset = 0;
163 163
164 gcmkONERROR(gckOS_GetPhysicalAddress( 164 gcmkONERROR(gckOS_GetPhysicalAddress(
165 Command->os, 165 Command->os,
166 Command->logical, 166 Command->logical,
167 &physical 167 &physical
168 )); 168 ));
169 169
170 gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical( 170 gcmkVERIFY_OK(gckOS_CPUPhysicalToGPUPhysical(
171 Command->os, 171 Command->os,
172 physical, 172 physical,
173 &physical 173 &physical
174 )); 174 ));
175 175
176 gcmkSAFECASTPHYSADDRT(Command->physical, physical); 176 gcmkSAFECASTPHYSADDRT(Command->physical, physical);
177 177
178 if (currentIndex != -1) 178 if (currentIndex != -1)
179 { 179 {
180 if (Stalled) 180 if (Stalled)
181 { 181 {
182 gckOS_Signal( 182 gckOS_Signal(
183 Command->os, 183 Command->os,
184 Command->queues[currentIndex].signal, 184 Command->queues[currentIndex].signal,
185 gcvTRUE 185 gcvTRUE
186 ); 186 );
187 } 187 }
188 else 188 else
189 { 189 {
190 /* Mark the command queue as available. */ 190 /* Mark the command queue as available. */
191 gcmkONERROR(gckEVENT_Signal( 191 gcmkONERROR(gckEVENT_Signal(
192 Command->kernel->eventObj, 192 Command->kernel->eventObj,
193 Command->queues[currentIndex].signal, 193 Command->queues[currentIndex].signal,
194 gcvKERNEL_COMMAND 194 gcvKERNEL_COMMAND
195 )); 195 ));
196 } 196 }
197 } 197 }
198 198
199 /* Success. */ 199 /* Success. */
200 gcmkFOOTER_ARG("Command->index=%d", Command->index); 200 gcmkFOOTER_ARG("Command->index=%d", Command->index);
201 return gcvSTATUS_OK; 201 return gcvSTATUS_OK;
202 202
203 OnError: 203 OnError:
204 /* Return the status. */ 204 /* Return the status. */
205 gcmkFOOTER(); 205 gcmkFOOTER();
206 return status; 206 return status;
207 } 207 }
208 208
209 gceSTATUS 209 gceSTATUS
210 _IncrementCommitAtom( 210 _IncrementCommitAtom(
211 IN gckCOMMAND Command, 211 IN gckCOMMAND Command,
212 IN gctBOOL Increment 212 IN gctBOOL Increment
213 ) 213 )
214 { 214 {
215 gceSTATUS status; 215 gceSTATUS status;
216 gckHARDWARE hardware; 216 gckHARDWARE hardware;
217 gctINT32 atomValue; 217 gctINT32 atomValue;
218 gctBOOL powerAcquired = gcvFALSE; 218 gctBOOL powerAcquired = gcvFALSE;
219 219
220 gcmkHEADER_ARG("Command=0x%x", Command); 220 gcmkHEADER_ARG("Command=0x%x", Command);
221 221
222 /* Extract the gckHARDWARE and gckEVENT objects. */ 222 /* Extract the gckHARDWARE and gckEVENT objects. */
223 hardware = Command->kernel->hardware; 223 hardware = Command->kernel->hardware;
224 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); 224 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
225 225
226 /* Grab the power mutex. */ 226 /* Grab the power mutex. */
227 gcmkONERROR(gckOS_AcquireMutex( 227 gcmkONERROR(gckOS_AcquireMutex(
228 Command->os, hardware->powerMutex, gcvINFINITE 228 Command->os, hardware->powerMutex, gcvINFINITE
229 )); 229 ));
230 powerAcquired = gcvTRUE; 230 powerAcquired = gcvTRUE;
231 231
232 /* Increment the commit atom. */ 232 /* Increment the commit atom. */
233 if (Increment) 233 if (Increment)
234 { 234 {
235 gcmkONERROR(gckOS_AtomIncrement( 235 gcmkONERROR(gckOS_AtomIncrement(
236 Command->os, Command->atomCommit, &atomValue 236 Command->os, Command->atomCommit, &atomValue
237 )); 237 ));
238 } 238 }
239 else 239 else
240 { 240 {
241 gcmkONERROR(gckOS_AtomDecrement( 241 gcmkONERROR(gckOS_AtomDecrement(
242 Command->os, Command->atomCommit, &atomValue 242 Command->os, Command->atomCommit, &atomValue
243 )); 243 ));
244 } 244 }
245 245
246 /* Release the power mutex. */ 246 /* Release the power mutex. */
247 gcmkONERROR(gckOS_ReleaseMutex( 247 gcmkONERROR(gckOS_ReleaseMutex(
248 Command->os, hardware->powerMutex 248 Command->os, hardware->powerMutex
249 )); 249 ));
250 powerAcquired = gcvFALSE; 250 powerAcquired = gcvFALSE;
251 251
252 /* Success. */ 252 /* Success. */
253 gcmkFOOTER(); 253 gcmkFOOTER();
254 return gcvSTATUS_OK; 254 return gcvSTATUS_OK;
255 255
256 OnError: 256 OnError:
257 if (powerAcquired) 257 if (powerAcquired)
258 { 258 {
259 /* Release the power mutex. */ 259 /* Release the power mutex. */
260 gcmkVERIFY_OK(gckOS_ReleaseMutex( 260 gcmkVERIFY_OK(gckOS_ReleaseMutex(
261 Command->os, hardware->powerMutex 261 Command->os, hardware->powerMutex
262 )); 262 ));
263 } 263 }
264 264
265 /* Return the status. */ 265 /* Return the status. */
266 gcmkFOOTER(); 266 gcmkFOOTER();
267 return status; 267 return status;
268 } 268 }
269 269
270 #if gcdSECURE_USER 270 #if gcdSECURE_USER
271 gceSTATUS 271 gceSTATUS
272 _ProcessHints( 272 _ProcessHints(
273 IN gckCOMMAND Command, 273 IN gckCOMMAND Command,
274 IN gctUINT32 ProcessID, 274 IN gctUINT32 ProcessID,
275 IN gcoCMDBUF CommandBuffer 275 IN gcoCMDBUF CommandBuffer
276 ) 276 )
277 { 277 {
278 gceSTATUS status = gcvSTATUS_OK; 278 gceSTATUS status = gcvSTATUS_OK;
279 gckKERNEL kernel; 279 gckKERNEL kernel;
280 gctBOOL needCopy = gcvFALSE; 280 gctBOOL needCopy = gcvFALSE;
281 gcskSECURE_CACHE_PTR cache; 281 gcskSECURE_CACHE_PTR cache;
282 gctUINT8_PTR commandBufferLogical; 282 gctUINT8_PTR commandBufferLogical;
283 gctUINT8_PTR hintedData; 283 gctUINT8_PTR hintedData;
284 gctUINT32_PTR hintArray; 284 gctUINT32_PTR hintArray;
285 gctUINT i, hintCount; 285 gctUINT i, hintCount;
286 286
287 gcmkHEADER_ARG( 287 gcmkHEADER_ARG(
288 "Command=0x%08X ProcessID=%d CommandBuffer=0x%08X", 288 "Command=0x%08X ProcessID=%d CommandBuffer=0x%08X",
289 Command, ProcessID, CommandBuffer 289 Command, ProcessID, CommandBuffer
290 ); 290 );
291 291
292 /* Verify the arguments. */ 292 /* Verify the arguments. */
293 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 293 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
294 294
295 /* Reset state array pointer. */ 295 /* Reset state array pointer. */
296 hintArray = gcvNULL; 296 hintArray = gcvNULL;
297 297
298 /* Get the kernel object. */ 298 /* Get the kernel object. */
299 kernel = Command->kernel; 299 kernel = Command->kernel;
300 300
301 /* Get the cache form the database. */ 301 /* Get the cache form the database. */
302 gcmkONERROR(gckKERNEL_GetProcessDBCache(kernel, ProcessID, &cache)); 302 gcmkONERROR(gckKERNEL_GetProcessDBCache(kernel, ProcessID, &cache));
303 303
304 /* Determine the start of the command buffer. */ 304 /* Determine the start of the command buffer. */
305 commandBufferLogical 305 commandBufferLogical
306 = (gctUINT8_PTR) CommandBuffer->logical 306 = (gctUINT8_PTR) CommandBuffer->logical
307 + CommandBuffer->startOffset; 307 + CommandBuffer->startOffset;
308 308
309 /* Determine the number of records in the state array. */ 309 /* Determine the number of records in the state array. */
310 hintCount = CommandBuffer->hintArrayTail - CommandBuffer->hintArray; 310 hintCount = CommandBuffer->hintArrayTail - CommandBuffer->hintArray;
311 311
312 /* Check wehther we need to copy the structures or not. */ 312 /* Check wehther we need to copy the structures or not. */
313 gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); 313 gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy));
314 314
315 /* Get access to the state array. */ 315 /* Get access to the state array. */
316 if (needCopy) 316 if (needCopy)
317 { 317 {
318 gctUINT copySize; 318 gctUINT copySize;
319 319
320 if (Command->hintArrayAllocated && 320 if (Command->hintArrayAllocated &&
321 (Command->hintArraySize < CommandBuffer->hintArraySize)) 321 (Command->hintArraySize < CommandBuffer->hintArraySize))
322 { 322 {
323 gcmkONERROR(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); 323 gcmkONERROR(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray)));
324 Command->hintArraySize = gcvFALSE; 324 Command->hintArraySize = gcvFALSE;
325 } 325 }
326 326
327 if (!Command->hintArrayAllocated) 327 if (!Command->hintArrayAllocated)
328 { 328 {
329 gctPOINTER pointer = gcvNULL; 329 gctPOINTER pointer = gcvNULL;
330 330
331 gcmkONERROR(gckOS_Allocate( 331 gcmkONERROR(gckOS_Allocate(
332 Command->os, 332 Command->os,
333 CommandBuffer->hintArraySize, 333 CommandBuffer->hintArraySize,
334 &pointer 334 &pointer
335 )); 335 ));
336 336
337 Command->hintArray = gcmPTR_TO_UINT64(pointer); 337 Command->hintArray = gcmPTR_TO_UINT64(pointer);
338 Command->hintArrayAllocated = gcvTRUE; 338 Command->hintArrayAllocated = gcvTRUE;
339 Command->hintArraySize = CommandBuffer->hintArraySize; 339 Command->hintArraySize = CommandBuffer->hintArraySize;
340 } 340 }
341 341
342 hintArray = gcmUINT64_TO_PTR(Command->hintArray); 342 hintArray = gcmUINT64_TO_PTR(Command->hintArray);
343 copySize = hintCount * gcmSIZEOF(gctUINT32); 343 copySize = hintCount * gcmSIZEOF(gctUINT32);
344 344
345 gcmkONERROR(gckOS_CopyFromUserData( 345 gcmkONERROR(gckOS_CopyFromUserData(
346 Command->os, 346 Command->os,
347 hintArray, 347 hintArray,
348 gcmUINT64_TO_PTR(CommandBuffer->hintArray), 348 gcmUINT64_TO_PTR(CommandBuffer->hintArray),
349 copySize 349 copySize
350 )); 350 ));
351 } 351 }
352 else 352 else
353 { 353 {
354 gctPOINTER pointer = gcvNULL; 354 gctPOINTER pointer = gcvNULL;
355 355
356 gcmkONERROR(gckOS_MapUserPointer( 356 gcmkONERROR(gckOS_MapUserPointer(
357 Command->os, 357 Command->os,
358 gcmUINT64_TO_PTR(CommandBuffer->hintArray), 358 gcmUINT64_TO_PTR(CommandBuffer->hintArray),
359 CommandBuffer->hintArraySize, 359 CommandBuffer->hintArraySize,
360 &pointer 360 &pointer
361 )); 361 ));
362 362
363 hintArray = pointer; 363 hintArray = pointer;
364 } 364 }
365 365
366 /* Scan through the buffer. */ 366 /* Scan through the buffer. */
367 for (i = 0; i < hintCount; i += 1) 367 for (i = 0; i < hintCount; i += 1)
368 { 368 {
369 /* Determine the location of the hinted data. */ 369 /* Determine the location of the hinted data. */
370 hintedData = commandBufferLogical + hintArray[i]; 370 hintedData = commandBufferLogical + hintArray[i];
371 371
372 /* Map handle into physical address. */ 372 /* Map handle into physical address. */
373 gcmkONERROR(gckKERNEL_MapLogicalToPhysical( 373 gcmkONERROR(gckKERNEL_MapLogicalToPhysical(
374 kernel, cache, (gctPOINTER) hintedData 374 kernel, cache, (gctPOINTER) hintedData
375 )); 375 ));
376 } 376 }
377 377
378 OnError: 378 OnError:
379 /* Get access to the state array. */ 379 /* Get access to the state array. */
380 if (!needCopy && (hintArray != gcvNULL)) 380 if (!needCopy && (hintArray != gcvNULL))
381 { 381 {
382 gcmkVERIFY_OK(gckOS_UnmapUserPointer( 382 gcmkVERIFY_OK(gckOS_UnmapUserPointer(
383 Command->os, 383 Command->os,
384 gcmUINT64_TO_PTR(CommandBuffer->hintArray), 384 gcmUINT64_TO_PTR(CommandBuffer->hintArray),
385 CommandBuffer->hintArraySize, 385 CommandBuffer->hintArraySize,
386 hintArray 386 hintArray
387 )); 387 ));
388 } 388 }
389 389
390 /* Return the status. */ 390 /* Return the status. */
391 gcmkFOOTER(); 391 gcmkFOOTER();
392 return status; 392 return status;
393 } 393 }
394 #endif 394 #endif
395 395
396 #if !gcdNULL_DRIVER 396 #if !gcdNULL_DRIVER
397 gceSTATUS 397 gceSTATUS
398 _FlushMMU( 398 _FlushMMU(
399 IN gckCOMMAND Command 399 IN gckCOMMAND Command
400 ) 400 )
401 { 401 {
402 #if gcdSECURITY 402 #if gcdSECURITY
403 return gcvSTATUS_OK; 403 return gcvSTATUS_OK;
404 #else 404 #else
405 gceSTATUS status; 405 gceSTATUS status;
406 gctUINT32 oldValue; 406 gctUINT32 oldValue;
407 gckHARDWARE hardware = Command->kernel->hardware; 407 gckHARDWARE hardware = Command->kernel->hardware;
408 gctBOOL pause = gcvFALSE; 408 gctBOOL pause = gcvFALSE;
409 409
410 gctUINT8_PTR pointer; 410 gctUINT8_PTR pointer;
411 gctUINT32 address; 411 gctUINT32 address;
412 gctUINT32 eventBytes; 412 gctUINT32 eventBytes;
413 gctUINT32 endBytes; 413 gctUINT32 endBytes;
414 gctUINT32 bufferSize; 414 gctUINT32 bufferSize;
415 gctUINT32 executeBytes; 415 gctUINT32 executeBytes;
416 gctUINT32 waitLinkBytes; 416 gctUINT32 waitLinkBytes;
417 417
418 gcmkONERROR(gckOS_AtomicExchange(Command->os, 418 gcmkONERROR(gckOS_AtomicExchange(Command->os,
419 hardware->pageTableDirty[gcvENGINE_RENDER], 419 hardware->pageTableDirty[gcvENGINE_RENDER],
420 0, 420 0,
421 &oldValue)); 421 &oldValue));
422 422
423 if (oldValue) 423 if (oldValue)
424 { 424 {
425 /* Page Table is upated, flush mmu before commit. */ 425 /* Page Table is upated, flush mmu before commit. */
426 gcmkONERROR(gckHARDWARE_FlushMMU(hardware)); 426 gcmkONERROR(gckHARDWARE_FlushMMU(hardware));
427 427
428 if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE) 428 if ((oldValue & gcvPAGE_TABLE_DIRTY_BIT_FE)
429 && (!hardware->stallFEPrefetch) 429 && (!hardware->stallFEPrefetch)
430 ) 430 )
431 { 431 {
432 pause = gcvTRUE; 432 pause = gcvTRUE;
433 } 433 }
434 } 434 }
435 435
436 if (pause) 436 if (pause)
437 { 437 {
438 /* Query size. */ 438 /* Query size. */
439 gcmkONERROR(gckHARDWARE_Event(hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes)); 439 gcmkONERROR(gckHARDWARE_Event(hardware, gcvNULL, 0, gcvKERNEL_PIXEL, &eventBytes));
440 gcmkONERROR(gckHARDWARE_End(hardware, gcvNULL, ~0U, &endBytes)); 440 gcmkONERROR(gckHARDWARE_End(hardware, gcvNULL, ~0U, &endBytes));
441 441
442 executeBytes = eventBytes + endBytes; 442 executeBytes = eventBytes + endBytes;
443 443
444 gcmkONERROR(gckHARDWARE_WaitLink( 444 gcmkONERROR(gckHARDWARE_WaitLink(
445 hardware, 445 hardware,
446 gcvNULL, 446 gcvNULL,
447 ~0U, 447 ~0U,
448 Command->offset + executeBytes, 448 Command->offset + executeBytes,
449 &waitLinkBytes, 449 &waitLinkBytes,
450 gcvNULL, 450 gcvNULL,
451 gcvNULL 451 gcvNULL
452 )); 452 ));
453 453
454 /* Reserve space. */ 454 /* Reserve space. */
455 gcmkONERROR(gckCOMMAND_Reserve( 455 gcmkONERROR(gckCOMMAND_Reserve(
456 Command, 456 Command,
457 executeBytes, 457 executeBytes,
458 (gctPOINTER *)&pointer, 458 (gctPOINTER *)&pointer,
459 &bufferSize 459 &bufferSize
460 )); 460 ));
461 461
462 /* Pointer to reserved address. */ 462 /* Pointer to reserved address. */
463 address = Command->address + Command->offset; 463 address = Command->address + Command->offset;
464 464
465 /* Append EVENT(29). */ 465 /* Append EVENT(29). */
466 gcmkONERROR(gckHARDWARE_Event( 466 gcmkONERROR(gckHARDWARE_Event(
467 hardware, 467 hardware,
468 pointer, 468 pointer,
469 29, 469 29,
470 gcvKERNEL_PIXEL, 470 gcvKERNEL_PIXEL,
471 &eventBytes 471 &eventBytes
472 )); 472 ));
473 473
474 /* Append END. */ 474 /* Append END. */
475 pointer += eventBytes; 475 pointer += eventBytes;
476 address += eventBytes; 476 address += eventBytes;
477 477
478 gcmkONERROR(gckHARDWARE_End(hardware, pointer, address, &endBytes)); 478 gcmkONERROR(gckHARDWARE_End(hardware, pointer, address, &endBytes));
479 479
480 #if USE_KERNEL_VIRTUAL_BUFFERS 480 #if USE_KERNEL_VIRTUAL_BUFFERS
481 if (hardware->kernel->virtualCommandBuffer) 481 if (hardware->kernel->virtualCommandBuffer)
482 { 482 {
483 gcmkONERROR(gckKERNEL_GetGPUAddress( 483 gcmkONERROR(gckKERNEL_GetGPUAddress(
484 hardware->kernel, 484 hardware->kernel,
485 pointer, 485 pointer,
486 gcvFALSE, 486 gcvFALSE,
487 Command->virtualMemory, 487 Command->virtualMemory,
488 &hardware->lastEnd 488 &hardware->lastEnd
489 )); 489 ));
490 } 490 }
491 #endif 491 #endif
492 492
493 gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes)); 493 gcmkONERROR(gckCOMMAND_Execute(Command, executeBytes));
494 } 494 }
495 495
496 return gcvSTATUS_OK; 496 return gcvSTATUS_OK;
497 OnError: 497 OnError:
498 return status; 498 return status;
499 #endif 499 #endif
500 } 500 }
501 501
502 gceSTATUS 502 gceSTATUS
503 _DummyDraw( 503 _DummyDraw(
504 IN gckCOMMAND Command 504 IN gckCOMMAND Command
505 ) 505 )
506 { 506 {
507 #if gcdSECURITY 507 #if gcdSECURITY
508 return gcvSTATUS_OK; 508 return gcvSTATUS_OK;
509 #else 509 #else
510 gceSTATUS status; 510 gceSTATUS status;
511 gckHARDWARE hardware = Command->kernel->hardware; 511 gckHARDWARE hardware = Command->kernel->hardware;
512 512
513 gctUINT8_PTR pointer; 513 gctUINT8_PTR pointer;
514 gctUINT32 bufferSize; 514 gctUINT32 bufferSize;
515 515
516 gctUINT32 dummyDrawBytes; 516 gctUINT32 dummyDrawBytes;
517 gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID; 517 gceDUMMY_DRAW_TYPE dummyDrawType = gcvDUMMY_DRAW_INVALID;
518 518
519 if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW)) 519 if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FE_NEED_DUMMYDRAW))
520 { 520 {
521 dummyDrawType = gcvDUMMY_DRAW_GC400; 521 dummyDrawType = gcvDUMMY_DRAW_GC400;
522 } 522 }
523 523
524 if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) && 524 if (!gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC_DEFER_FILL_FIX) &&
525 gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC)) 525 gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_USC))
526 { 526 {
527 dummyDrawType = gcvDUMMY_DRAW_V60; 527 dummyDrawType = gcvDUMMY_DRAW_V60;
528 } 528 }
529 529
530 if (dummyDrawType != gcvDUMMY_DRAW_INVALID) 530 if (dummyDrawType != gcvDUMMY_DRAW_INVALID)
531 { 531 {
532 gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); 532 gckHARDWARE_DummyDraw(hardware, gcvNULL, Command->queues[0].address, dummyDrawType, &dummyDrawBytes);
533 533
534 /* Reserve space. */ 534 /* Reserve space. */
535 gcmkONERROR(gckCOMMAND_Reserve( 535 gcmkONERROR(gckCOMMAND_Reserve(
536 Command, 536 Command,
537 dummyDrawBytes, 537 dummyDrawBytes,
538 (gctPOINTER *)&pointer, 538 (gctPOINTER *)&pointer,
539 &bufferSize 539 &bufferSize
540 )); 540 ));
541 541
542 gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes); 542 gckHARDWARE_DummyDraw(hardware, pointer, Command->queues[0].address, dummyDrawType, &dummyDrawBytes);
543 543
544 gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes)); 544 gcmkONERROR(gckCOMMAND_Execute(Command, dummyDrawBytes));
545 } 545 }
546 546
547 return gcvSTATUS_OK; 547 return gcvSTATUS_OK;
548 OnError: 548 OnError:
549 return status; 549 return status;
550 #endif 550 #endif
551 } 551 }
552 552
553 #endif 553 #endif
554 554
555 static void 555 static void
556 _DumpBuffer( 556 _DumpBuffer(
557 IN gctPOINTER Buffer, 557 IN gctPOINTER Buffer,
558 IN gctUINT32 GpuAddress, 558 IN gctUINT32 GpuAddress,
559 IN gctSIZE_T Size 559 IN gctSIZE_T Size
560 ) 560 )
561 { 561 {
562 gctSIZE_T i, line, left; 562 gctSIZE_T i, line, left;
563 gctUINT32_PTR data = Buffer; 563 gctUINT32_PTR data = Buffer;
564 564
565 line = Size / 32; 565 line = Size / 32;
566 left = Size % 32; 566 left = Size % 32;
567 567
568 for (i = 0; i < line; i++) 568 for (i = 0; i < line; i++)
569 { 569 {
570 gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X", 570 gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X %08X",
571 GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); 571 GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]);
572 data += 8; 572 data += 8;
573 GpuAddress += 8 * 4; 573 GpuAddress += 8 * 4;
574 } 574 }
575 575
576 switch(left) 576 switch(left)
577 { 577 {
578 case 28: 578 case 28:
579 gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X", 579 gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X %08X",
580 GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]); 580 GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5], data[6]);
581 break; 581 break;
582 case 24: 582 case 24:
583 gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X", 583 gcmkPRINT("%08X : %08X %08X %08X %08X %08X %08X",
584 GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]); 584 GpuAddress, data[0], data[1], data[2], data[3], data[4], data[5]);
585 break; 585 break;
586 case 20: 586 case 20:
587 gcmkPRINT("%08X : %08X %08X %08X %08X %08X", 587 gcmkPRINT("%08X : %08X %08X %08X %08X %08X",
588 GpuAddress, data[0], data[1], data[2], data[3], data[4]); 588 GpuAddress, data[0], data[1], data[2], data[3], data[4]);
589 break; 589 break;
590 case 16: 590 case 16:
591 gcmkPRINT("%08X : %08X %08X %08X %08X", 591 gcmkPRINT("%08X : %08X %08X %08X %08X",
592 GpuAddress, data[0], data[1], data[2], data[3]); 592 GpuAddress, data[0], data[1], data[2], data[3]);
593 break; 593 break;
594 case 12: 594 case 12:
595 gcmkPRINT("%08X : %08X %08X %08X", 595 gcmkPRINT("%08X : %08X %08X %08X",
596 GpuAddress, data[0], data[1], data[2]); 596 GpuAddress, data[0], data[1], data[2]);
597 break; 597 break;
598 case 8: 598 case 8:
599 gcmkPRINT("%08X : %08X %08X", 599 gcmkPRINT("%08X : %08X %08X",
600 GpuAddress, data[0], data[1]); 600 GpuAddress, data[0], data[1]);
601 break; 601 break;
602 case 4: 602 case 4:
603 gcmkPRINT("%08X : %08X", 603 gcmkPRINT("%08X : %08X",
604 GpuAddress, data[0]); 604 GpuAddress, data[0]);
605 break; 605 break;
606 default: 606 default:
607 break; 607 break;
608 } 608 }
609 } 609 }
610 610
611 void 611 void
612 _DumpKernelCommandBuffer( 612 _DumpKernelCommandBuffer(
613 IN gckCOMMAND Command 613 IN gckCOMMAND Command
614 ) 614 )
615 { 615 {
616 gctINT i; 616 gctINT i;
617 gctUINT64 physical = 0; 617 gctUINT64 physical = 0;
618 gctUINT32 address; 618 gctUINT32 address;
619 gctPOINTER entry = gcvNULL; 619 gctPOINTER entry = gcvNULL;
620 620
621 for (i = 0; i < gcdCOMMAND_QUEUES; i++) 621 for (i = 0; i < gcdCOMMAND_QUEUES; i++)
622 { 622 {
623 entry = Command->queues[i].logical; 623 entry = Command->queues[i].logical;
624 624
625 gckOS_GetPhysicalAddress(Command->os, entry, &physical); 625 gckOS_GetPhysicalAddress(Command->os, entry, &physical);
626 626
627 gckOS_CPUPhysicalToGPUPhysical(Command->os, physical, &physical); 627 gckOS_CPUPhysicalToGPUPhysical(Command->os, physical, &physical);
628 628
629 gcmkPRINT("Kernel command buffer %d\n", i); 629 gcmkPRINT("Kernel command buffer %d\n", i);
630 630
631 gcmkSAFECASTPHYSADDRT(address, physical); 631 gcmkSAFECASTPHYSADDRT(address, physical);
632 632
633 _DumpBuffer(entry, address, Command->pageSize); 633 _DumpBuffer(entry, address, Command->pageSize);
634 } 634 }
635 } 635 }
636 636
637 #if !gcdNULL_DRIVER 637 #if !gcdNULL_DRIVER
638 gceSTATUS 638 gceSTATUS
639 _HandlePatchList( 639 _HandlePatchList(
640 IN gckCOMMAND Command, 640 IN gckCOMMAND Command,
641 IN gcoCMDBUF CommandBuffer, 641 IN gcoCMDBUF CommandBuffer,
642 IN gctBOOL NeedCopy, 642 IN gctBOOL NeedCopy,
643 OUT gctUINT64 *AsyncCommandStamp 643 OUT gctUINT64 *AsyncCommandStamp
644 ) 644 )
645 { 645 {
646 gceSTATUS status; 646 gceSTATUS status;
647 gcsPATCH_LIST * uList; 647 gcsPATCH_LIST * uList;
648 gcsPATCH_LIST * previous; 648 gcsPATCH_LIST * previous;
649 gcsPATCH_LIST * kList; 649 gcsPATCH_LIST * kList;
650 gctUINT64 asyncStamp = 0; 650 gctUINT64 asyncStamp = 0;
651 651
652 gcmkHEADER_ARG( 652 gcmkHEADER_ARG(
653 "Command=0x%x CommandBuffer=0x%x NeedCopy=%d", 653 "Command=0x%x CommandBuffer=0x%x NeedCopy=%d",
654 Command, CommandBuffer, NeedCopy 654 Command, CommandBuffer, NeedCopy
655 ); 655 );
656 656
657 uList = gcmUINT64_TO_PTR(CommandBuffer->patchHead); 657 uList = gcmUINT64_TO_PTR(CommandBuffer->patchHead);
658 658
659 while (uList) 659 while (uList)
660 { 660 {
661 gctUINT i; 661 gctUINT i;
662 662
663 kList = gcvNULL; 663 kList = gcvNULL;
664 previous = uList; 664 previous = uList;
665 665
666 gcmkONERROR(gckKERNEL_OpenUserData( 666 gcmkONERROR(gckKERNEL_OpenUserData(
667 Command->kernel, 667 Command->kernel,
668 NeedCopy, 668 NeedCopy,
669 Command->kList, 669 Command->kList,
670 uList, 670 uList,
671 gcmSIZEOF(gcsPATCH_LIST), 671 gcmSIZEOF(gcsPATCH_LIST),
672 (gctPOINTER *)&kList 672 (gctPOINTER *)&kList
673 )); 673 ));
674 674
675 for (i = 0; i < kList->count; i++) 675 for (i = 0; i < kList->count; i++)
676 { 676 {
677 gctUINT64 stamp = 0; 677 gctUINT64 stamp = 0;
678 gcsPATCH * patch = &kList->patch[i]; 678 gcsPATCH * patch = &kList->patch[i];
679 679
680 /* Touch video memory node. */ 680 /* Touch video memory node. */
681 gcmkVERIFY_OK(gckVIDMEM_SetCommitStamp(Command->kernel, gcvENGINE_RENDER, patch->handle, Command->commitStamp)); 681 gcmkVERIFY_OK(gckVIDMEM_SetCommitStamp(Command->kernel, gcvENGINE_RENDER, patch->handle, Command->commitStamp));
682 682
683 /* Get stamp touched async command buffer. */ 683 /* Get stamp touched async command buffer. */
684 gcmkVERIFY_OK(gckVIDMEM_GetCommitStamp(Command->kernel, gcvENGINE_BLT, patch->handle, &stamp)); 684 gcmkVERIFY_OK(gckVIDMEM_GetCommitStamp(Command->kernel, gcvENGINE_BLT, patch->handle, &stamp));
685 685
686 /* Find latest one. */ 686 /* Find latest one. */
687 asyncStamp = gcmMAX(asyncStamp, stamp); 687 asyncStamp = gcmMAX(asyncStamp, stamp);
688 } 688 }
689 689
690 uList = kList->next; 690 uList = kList->next;
691 691
692 gcmkVERIFY_OK(gckKERNEL_CloseUserData( 692 gcmkVERIFY_OK(gckKERNEL_CloseUserData(
693 Command->kernel, 693 Command->kernel,
694 NeedCopy, 694 NeedCopy,
695 gcvFALSE, 695 gcvFALSE,
696 previous, 696 previous,
697 gcmSIZEOF(gcsPATCH_LIST), 697 gcmSIZEOF(gcsPATCH_LIST),
698 (gctPOINTER *)&kList 698 (gctPOINTER *)&kList
699 )); 699 ));
700 } 700 }
701 701
702 if ((Command->asyncCommand != gcvNULL) 702 if ((Command->asyncCommand != gcvNULL)
703 && (*(gctUINT64 *)Command->asyncCommand->fence->logical > asyncStamp) 703 && (*(gctUINT64 *)Command->asyncCommand->fence->logical > asyncStamp)
704 ) 704 )
705 { 705 {
706 /* No need to wait for async command buffer. */ 706 /* No need to wait for async command buffer. */
707 *AsyncCommandStamp = 0; 707 *AsyncCommandStamp = 0;
708 } 708 }
709 else 709 else
710 { 710 {
711 /* Need to add a fence wait. */ 711 /* Need to add a fence wait. */
712 *AsyncCommandStamp = asyncStamp; 712 *AsyncCommandStamp = asyncStamp;
713 } 713 }
714 714
715 gcmkFOOTER_NO(); 715 gcmkFOOTER_NO();
716 return gcvSTATUS_OK; 716 return gcvSTATUS_OK;
717 717
718 OnError: 718 OnError:
719 if (kList) 719 if (kList)
720 { 720 {
721 gcmkVERIFY_OK(gckKERNEL_CloseUserData( 721 gcmkVERIFY_OK(gckKERNEL_CloseUserData(
722 Command->kernel, 722 Command->kernel,
723 NeedCopy, 723 NeedCopy,
724 gcvFALSE, 724 gcvFALSE,
725 previous, 725 previous,
726 gcmSIZEOF(gcsPATCH_LIST), 726 gcmSIZEOF(gcsPATCH_LIST),
727 (gctPOINTER *)&kList 727 (gctPOINTER *)&kList
728 )); 728 ));
729 } 729 }
730 730
731 gcmkFOOTER(); 731 gcmkFOOTER();
732 return status; 732 return status;
733 } 733 }
734 734
735 gceSTATUS 735 gceSTATUS
736 _WaitForAsyncCommandStamp( 736 _WaitForAsyncCommandStamp(
737 IN gckCOMMAND Command, 737 IN gckCOMMAND Command,
738 IN gctUINT64 Stamp 738 IN gctUINT64 Stamp
739 ) 739 )
740 { 740 {
741 gctUINT32 bytes; 741 gctUINT32 bytes;
742 gceSTATUS status; 742 gceSTATUS status;
743 gctUINT32 fenceAddress; 743 gctUINT32 fenceAddress;
744 gctUINT32 bufferSize; 744 gctUINT32 bufferSize;
745 gctPOINTER pointer; 745 gctPOINTER pointer;
746 gcmkHEADER_ARG("Stamp = 0x%llx", Stamp); 746 gcmkHEADER_ARG("Stamp = 0x%llx", Stamp);
747 747
748 fenceAddress = Command->asyncCommand->fence->address; 748 fenceAddress = Command->asyncCommand->fence->address;
749 749
750 gcmkONERROR(gckHARDWARE_WaitFence(Command->kernel->hardware, 750 gcmkONERROR(gckHARDWARE_WaitFence(Command->kernel->hardware,
751 gcvNULL, 751 gcvNULL,
752 Stamp, 752 Stamp,
753 fenceAddress, 753 fenceAddress,
754 &bytes 754 &bytes
755 )); 755 ));
756 756
757 gcmkONERROR(gckCOMMAND_Reserve( 757 gcmkONERROR(gckCOMMAND_Reserve(
758 Command, 758 Command,
759 bytes, 759 bytes,
760 &pointer, 760 &pointer,
761 &bufferSize 761 &bufferSize
762 )); 762 ));
763 763
764 gcmkONERROR(gckHARDWARE_WaitFence( 764 gcmkONERROR(gckHARDWARE_WaitFence(
765 Command->kernel->hardware, 765 Command->kernel->hardware,
766 pointer, 766 pointer,
767 Stamp, 767 Stamp,
768 fenceAddress, 768 fenceAddress,
769 &bytes 769 &bytes
770 )); 770 ));
771 771
772 gcmkONERROR(gckCOMMAND_Execute(Command, bytes)); 772 gcmkONERROR(gckCOMMAND_Execute(Command, bytes));
773 773
774 gcmkFOOTER_NO(); 774 gcmkFOOTER_NO();
775 return gcvSTATUS_OK; 775 return gcvSTATUS_OK;
776 776
777 OnError: 777 OnError:
778 gcmkFOOTER(); 778 gcmkFOOTER();
779 return status; 779 return status;
780 } 780 }
781 781
782 /******************************************************************************\ 782 /******************************************************************************\
783 **************** Helper functions for parsing gcoCMDBUF ************************ 783 **************** Helper functions for parsing gcoCMDBUF ************************
784 \******************************************************************************/ 784 \******************************************************************************/
785 static void 785 static void
786 _GetCMDBUFSize( 786 _GetCMDBUFSize(
787 IN gcoCMDBUF CommandBuffer, 787 IN gcoCMDBUF CommandBuffer,
788 OUT gctUINT_PTR CommandBufferSize 788 OUT gctUINT_PTR CommandBufferSize
789 ) 789 )
790 { 790 {
791 *CommandBufferSize 791 *CommandBufferSize
792 = CommandBuffer->offset 792 = CommandBuffer->offset
793 + CommandBuffer->reservedTail 793 + CommandBuffer->reservedTail
794 - CommandBuffer->startOffset; 794 - CommandBuffer->startOffset;
795 } 795 }
796 796
797 static void 797 static void
798 _GetCMDBUFTail( 798 _GetCMDBUFTail(
799 IN gcoCMDBUF CommandBuffer, 799 IN gcoCMDBUF CommandBuffer,
800 OUT gctUINT8_PTR * Tail 800 OUT gctUINT8_PTR * Tail
801 ) 801 )
802 { 802 {
803 gctUINT8_PTR commandBufferLogical; 803 gctUINT8_PTR commandBufferLogical;
804 gctUINT commandBufferSize; 804 gctUINT commandBufferSize;
805 805
806 commandBufferLogical 806 commandBufferLogical
807 = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) 807 = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical)
808 + CommandBuffer->startOffset; 808 + CommandBuffer->startOffset;
809 809
810 _GetCMDBUFSize(CommandBuffer, &commandBufferSize); 810 _GetCMDBUFSize(CommandBuffer, &commandBufferSize);
811 811
812 *Tail 812 *Tail
813 = commandBufferLogical 813 = commandBufferLogical
814 + commandBufferSize 814 + commandBufferSize
815 - CommandBuffer->reservedTail; 815 - CommandBuffer->reservedTail;
816 } 816 }
817 817
818 static void 818 static void
819 _ParseCMDBUFTail( 819 _ParseCMDBUFTail(
820 IN gckHARDWARE Hardware, 820 IN gckHARDWARE Hardware,
821 IN gcoCMDBUF CommandBuffer, 821 IN gcoCMDBUF CommandBuffer,
822 OUT gctUINT8_PTR * Fence, 822 OUT gctUINT8_PTR * Fence,
823 OUT gctUINT8_PTR * Link 823 OUT gctUINT8_PTR * Link
824 ) 824 )
825 { 825 {
826 gctUINT8_PTR tail; 826 gctUINT8_PTR tail;
827 827
828 _GetCMDBUFTail(CommandBuffer, &tail); 828 _GetCMDBUFTail(CommandBuffer, &tail);
829 829
830 if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_FENCE_64BIT)) 830 if (gckHARDWARE_IsFeatureAvailable(Hardware, gcvFEATURE_FENCE_64BIT))
831 { 831 {
832 *Fence = tail; 832 *Fence = tail;
833 *Link = tail + gcdRENDER_FENCE_LENGTH; 833 *Link = tail + gcdRENDER_FENCE_LENGTH;
834 } 834 }
835 else 835 else
836 { 836 {
837 *Fence = gcvNULL; 837 *Fence = gcvNULL;
838 *Link = tail; 838 *Link = tail;
839 } 839 }
840 } 840 }
841 841
842 gceSTATUS 842 gceSTATUS
843 _GetCMDBUFEntry( 843 _GetCMDBUFEntry(
844 IN gckCOMMAND Command, 844 IN gckCOMMAND Command,
845 IN gcoCMDBUF CommandBuffer, 845 IN gcoCMDBUF CommandBuffer,
846 OUT gctUINT32_PTR EntryAddress, 846 OUT gctUINT32_PTR EntryAddress,
847 OUT gctUINT32_PTR EntryBytes 847 OUT gctUINT32_PTR EntryBytes
848 ) 848 )
849 { 849 {
850 gceSTATUS status; 850 gceSTATUS status;
851 gctUINT8_PTR commandBufferLogical; 851 gctUINT8_PTR commandBufferLogical;
852 gctUINT commandBufferSize; 852 gctUINT commandBufferSize;
853 gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer; 853 gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer;
854 gctUINT32 commandBufferAddress; 854 gctUINT32 commandBufferAddress;
855 gctUINT offset; 855 gctUINT offset;
856 856
857 commandBufferLogical 857 commandBufferLogical
858 = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical) 858 = (gctUINT8_PTR) gcmUINT64_TO_PTR(CommandBuffer->logical)
859 + CommandBuffer->startOffset; 859 + CommandBuffer->startOffset;
860 860
861 /* Get the hardware address. */ 861 /* Get the hardware address. */
862 if (Command->kernel->virtualCommandBuffer) 862 if (Command->kernel->virtualCommandBuffer)
863 { 863 {
864 gckKERNEL kernel = Command->kernel; 864 gckKERNEL kernel = Command->kernel;
865 865
866 virtualCommandBuffer = gcmNAME_TO_PTR(CommandBuffer->physical); 866 virtualCommandBuffer = gcmNAME_TO_PTR(CommandBuffer->physical);
867 867
868 if (virtualCommandBuffer == gcvNULL) 868 if (virtualCommandBuffer == gcvNULL)
869 { 869 {
870 gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); 870 gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
871 } 871 }
872 872
873 gcmkONERROR(gckKERNEL_GetGPUAddress( 873 gcmkONERROR(gckKERNEL_GetGPUAddress(
874 Command->kernel, 874 Command->kernel,
875 commandBufferLogical, 875 commandBufferLogical,
876 gcvTRUE, 876 gcvTRUE,
877 virtualCommandBuffer, 877 virtualCommandBuffer,
878 &commandBufferAddress 878 &commandBufferAddress
879 )); 879 ));
880 } 880 }
881 else 881 else
882 { 882 {
883 gcmkONERROR(gckHARDWARE_ConvertLogical( 883 gcmkONERROR(gckHARDWARE_ConvertLogical(
884 Command->kernel->hardware, 884 Command->kernel->hardware,
885 commandBufferLogical, 885 commandBufferLogical,
886 gcvTRUE, 886 gcvTRUE,
887 &commandBufferAddress 887 &commandBufferAddress
888 )); 888 ));
889 } 889 }
890 890
891 /* Get offset. */ 891 /* Get offset. */
892 gcmkONERROR(gckHARDWARE_PipeSelect( 892 gcmkONERROR(gckHARDWARE_PipeSelect(
893 Command->kernel->hardware, gcvNULL, gcvPIPE_3D, &offset 893 Command->kernel->hardware, gcvNULL, gcvPIPE_3D, &offset
894 )); 894 ));
895 895
896 _GetCMDBUFSize(CommandBuffer, &commandBufferSize); 896 _GetCMDBUFSize(CommandBuffer, &commandBufferSize);
897 897
898 *EntryAddress = commandBufferAddress + offset; 898 *EntryAddress = commandBufferAddress + offset;
899 *EntryBytes = commandBufferSize - offset; 899 *EntryBytes = commandBufferSize - offset;
900 900
901 return gcvSTATUS_OK; 901 return gcvSTATUS_OK;
902 902
903 OnError: 903 OnError:
904 return status; 904 return status;
905 } 905 }
906 906
907 /******************************************************************************* 907 /*******************************************************************************
908 ** 908 **
909 ** Link a list of command buffer together to make them atomic. 909 ** Link a list of command buffer together to make them atomic.
910 ** Fence will be added in the last command buffer. 910 ** Fence will be added in the last command buffer.
911 */ 911 */
912 gceSTATUS 912 gceSTATUS
913 _ProcessUserCommandBufferList( 913 _ProcessUserCommandBufferList(
914 IN gckCOMMAND Command, 914 IN gckCOMMAND Command,
915 IN gcoCMDBUF CommandBufferListHead, 915 IN gcoCMDBUF CommandBufferListHead,
916 OUT gcoCMDBUF * CommandBufferListTail 916 OUT gcoCMDBUF * CommandBufferListTail
917 ) 917 )
918 { 918 {
919 gceSTATUS status; 919 gceSTATUS status;
920 gctBOOL needCopy; 920 gctBOOL needCopy;
921 921
922 gcoCMDBUF currentCMDBUF; 922 gcoCMDBUF currentCMDBUF;
923 gcoCMDBUF currentCMDBUFUser = CommandBufferListHead; 923 gcoCMDBUF currentCMDBUFUser = CommandBufferListHead;
924 924
925 gckOS_QueryNeedCopy(Command->os, 0, &needCopy); 925 gckOS_QueryNeedCopy(Command->os, 0, &needCopy);
926 926
927 /* Open first gcoCMDBUF object as currentCMDBUF. */ 927 /* Open first gcoCMDBUF object as currentCMDBUF. */
928 gcmkONERROR(gckKERNEL_OpenUserData( 928 gcmkONERROR(gckKERNEL_OpenUserData(
929 Command->kernel, 929 Command->kernel,
930 needCopy, 930 needCopy,
931 &Command->_commandBufferObject, 931 &Command->_commandBufferObject,
932 currentCMDBUFUser, 932 currentCMDBUFUser,
933 gcmSIZEOF(struct _gcoCMDBUF), 933 gcmSIZEOF(struct _gcoCMDBUF),
934 (gctPOINTER *)&currentCMDBUF 934 (gctPOINTER *)&currentCMDBUF
935 )); 935 ));
936 936
937 /* Iterate the list. */ 937 /* Iterate the list. */
938 while (currentCMDBUF->nextCMDBUF != 0) 938 while (currentCMDBUF->nextCMDBUF != 0)
939 { 939 {
940 gcoCMDBUF nextCMDBUFUser; 940 gcoCMDBUF nextCMDBUFUser;
941 gcoCMDBUF nextCMDBUF; 941 gcoCMDBUF nextCMDBUF;
942 gctUINT8_PTR fenceLogical = gcvNULL; 942 gctUINT8_PTR fenceLogical = gcvNULL;
943 gctUINT8_PTR linkLogical; 943 gctUINT8_PTR linkLogical;
944 gctUINT32 linkBytes = 8; 944 gctUINT32 linkBytes = 8;
945 gctUINT32 linkLow; 945 gctUINT32 linkLow;
946 gctUINT32 linkHigh; 946 gctUINT32 linkHigh;
947 947
948 gctUINT32 entryAddress = 0; 948 gctUINT32 entryAddress = 0;
949 gctUINT32 entryBytes = 0; 949 gctUINT32 entryBytes = 0;
950 950
951 nextCMDBUFUser 951 nextCMDBUFUser
952 = gcmUINT64_TO_PTR(currentCMDBUF->nextCMDBUF); 952 = gcmUINT64_TO_PTR(currentCMDBUF->nextCMDBUF);
953 953
954 /* Open next gcoCMDBUF object as nextCMDBUF. */ 954 /* Open next gcoCMDBUF object as nextCMDBUF. */
955 gcmkONERROR(gckKERNEL_OpenUserData( 955 gcmkONERROR(gckKERNEL_OpenUserData(
956 Command->kernel, 956 Command->kernel,
957 needCopy, 957 needCopy,
958 &Command->_nextCMDBUF, 958 &Command->_nextCMDBUF,
959 nextCMDBUFUser, 959 nextCMDBUFUser,
960 gcmSIZEOF(struct _gcoCMDBUF), 960 gcmSIZEOF(struct _gcoCMDBUF),
961 (gctPOINTER *)&nextCMDBUF 961 (gctPOINTER *)&nextCMDBUF
962 )); 962 ));
963 963
964 /* Get the start hardware address of nextCMDBUF. */ 964 /* Get the start hardware address of nextCMDBUF. */
965 gcmkONERROR(_GetCMDBUFEntry(Command, 965 gcmkONERROR(_GetCMDBUFEntry(Command,
966 nextCMDBUF, 966 nextCMDBUF,
967 &entryAddress, 967 &entryAddress,
968 &entryBytes 968 &entryBytes
969 )); 969 ));
970 970
971 /* Process current gcoCMDBUF object. */ 971 /* Process current gcoCMDBUF object. */
972 _ParseCMDBUFTail( 972 _ParseCMDBUFTail(
973 Command->kernel->hardware, 973 Command->kernel->hardware,
974 currentCMDBUF, 974 currentCMDBUF,
975 &fenceLogical, 975 &fenceLogical,
976 &linkLogical 976 &linkLogical
977 ); 977 );
978 978
979 /* Don't send fence in the middle of gcoCMDBUF list. */ 979 /* Don't send fence in the middle of gcoCMDBUF list. */
980 if (fenceLogical != gcvNULL) 980 if (fenceLogical != gcvNULL)
981 { 981 {
982 gctUINT i = gcdRENDER_FENCE_LENGTH / gcmSIZEOF(gctUINT32) / 2; 982 gctUINT i = gcdRENDER_FENCE_LENGTH / gcmSIZEOF(gctUINT32) / 2;
983 983
984 /* Fill NOPs in space reserved for fence. */ 984 /* Fill NOPs in space reserved for fence. */
985 while (i--) 985 while (i--)
986 { 986 {
987 gctSIZE_T nopBytes = 8; 987 gctSIZE_T nopBytes = 8;
988 gcmkONERROR(gckHARDWARE_Nop(Command->kernel->hardware, fenceLogical, &nopBytes)); 988 gcmkONERROR(gckHARDWARE_Nop(Command->kernel->hardware, fenceLogical, &nopBytes));
989 fenceLogical += nopBytes; 989 fenceLogical += nopBytes;
990 } 990 }
991 } 991 }
992 992
993 /* Generate a LINK from the end of current command buffer 993 /* Generate a LINK from the end of current command buffer
994 ** to the start of next command buffer. */ 994 ** to the start of next command buffer. */
995 gcmkONERROR(gckHARDWARE_Link( 995 gcmkONERROR(gckHARDWARE_Link(
996 Command->kernel->hardware, 996 Command->kernel->hardware,
997 linkLogical, 997 linkLogical,
998 entryAddress, 998 entryAddress,
999 entryBytes, 999 entryBytes,
1000 &linkBytes, 1000 &linkBytes,
1001 &linkLow, 1001 &linkLow,
1002 &linkHigh 1002 &linkHigh
1003 )); 1003 ));
1004 1004
1005 /* Close current gcoCMDBUF object which is processed. */ 1005 /* Close current gcoCMDBUF object which is processed. */
1006 gcmkVERIFY_OK(gckKERNEL_CloseUserData( 1006 gcmkVERIFY_OK(gckKERNEL_CloseUserData(
1007 Command->kernel, 1007 Command->kernel,
1008 needCopy, 1008 needCopy,
1009 gcvFALSE, 1009 gcvFALSE,
1010 currentCMDBUFUser, 1010 currentCMDBUFUser,
1011 gcmSIZEOF(struct _gcoCMDBUF), 1011 gcmSIZEOF(struct _gcoCMDBUF),
1012 (gctPOINTER *)&currentCMDBUF 1012 (gctPOINTER *)&currentCMDBUF
1013 )); 1013 ));
1014 1014
1015 /* Advance to next gcoCMDBUF object. */ 1015 /* Advance to next gcoCMDBUF object. */
1016 currentCMDBUFUser = nextCMDBUFUser; 1016 currentCMDBUFUser = nextCMDBUFUser;
1017 currentCMDBUF = nextCMDBUF; 1017 currentCMDBUF = nextCMDBUF;
1018 } 1018 }
1019 1019
1020 gcmkVERIFY_OK(gckKERNEL_CloseUserData( 1020 gcmkVERIFY_OK(gckKERNEL_CloseUserData(
1021 Command->kernel, 1021 Command->kernel,
1022 needCopy, 1022 needCopy,
1023 gcvFALSE, 1023 gcvFALSE,
1024 currentCMDBUFUser, 1024 currentCMDBUFUser,
1025 gcmSIZEOF(struct _gcoCMDBUF), 1025 gcmSIZEOF(struct _gcoCMDBUF),
1026 (gctPOINTER *)&currentCMDBUF 1026 (gctPOINTER *)&currentCMDBUF
1027 )); 1027 ));
1028 1028
1029 /* Return the tail of the list. */ 1029 /* Return the tail of the list. */
1030 *CommandBufferListTail = currentCMDBUFUser; 1030 *CommandBufferListTail = currentCMDBUFUser;
1031 1031
1032 return gcvSTATUS_OK; 1032 return gcvSTATUS_OK;
1033 1033
1034 OnError: 1034 OnError:
1035 return status; 1035 return status;
1036 } 1036 }
1037 #endif 1037 #endif
1038 1038
1039 /******************************************************************************\ 1039 /******************************************************************************\
1040 ****************************** gckCOMMAND API Code ****************************** 1040 ****************************** gckCOMMAND API Code ******************************
1041 \******************************************************************************/ 1041 \******************************************************************************/
1042 1042
1043 /******************************************************************************* 1043 /*******************************************************************************
1044 ** 1044 **
1045 ** gckCOMMAND_Construct 1045 ** gckCOMMAND_Construct
1046 ** 1046 **
1047 ** Construct a new gckCOMMAND object. 1047 ** Construct a new gckCOMMAND object.
1048 ** 1048 **
1049 ** INPUT: 1049 ** INPUT:
1050 ** 1050 **
1051 ** gckKERNEL Kernel 1051 ** gckKERNEL Kernel
1052 ** Pointer to an gckKERNEL object. 1052 ** Pointer to an gckKERNEL object.
1053 ** 1053 **
1054 ** OUTPUT: 1054 ** OUTPUT:
1055 ** 1055 **
1056 ** gckCOMMAND * Command 1056 ** gckCOMMAND * Command
1057 ** Pointer to a variable that will hold the pointer to the gckCOMMAND 1057 ** Pointer to a variable that will hold the pointer to the gckCOMMAND
1058 ** object. 1058 ** object.
1059 */ 1059 */
1060 gceSTATUS 1060 gceSTATUS
1061 gckCOMMAND_Construct( 1061 gckCOMMAND_Construct(
1062 IN gckKERNEL Kernel, 1062 IN gckKERNEL Kernel,
1063 OUT gckCOMMAND * Command 1063 OUT gckCOMMAND * Command
1064 ) 1064 )
1065 { 1065 {
1066 gckOS os; 1066 gckOS os;
1067 gckCOMMAND command = gcvNULL; 1067 gckCOMMAND command = gcvNULL;
1068 gceSTATUS status; 1068 gceSTATUS status;
1069 gctINT i; 1069 gctINT i;
1070 gctPOINTER pointer = gcvNULL; 1070 gctPOINTER pointer = gcvNULL;
1071 gctSIZE_T pageSize; 1071 gctSIZE_T pageSize;
1072 1072
1073 gcmkHEADER_ARG("Kernel=0x%x", Kernel); 1073 gcmkHEADER_ARG("Kernel=0x%x", Kernel);
1074 1074
1075 /* Verify the arguments. */ 1075 /* Verify the arguments. */
1076 gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); 1076 gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
1077 gcmkVERIFY_ARGUMENT(Command != gcvNULL); 1077 gcmkVERIFY_ARGUMENT(Command != gcvNULL);
1078 1078
1079 /* Extract the gckOS object. */ 1079 /* Extract the gckOS object. */
1080 os = Kernel->os; 1080 os = Kernel->os;
1081 1081
1082 /* Allocate the gckCOMMAND structure. */ 1082 /* Allocate the gckCOMMAND structure. */
1083 gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer)); 1083 gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckCOMMAND), &pointer));
1084 command = pointer; 1084 command = pointer;
1085 1085
1086 /* Reset the entire object. */ 1086 /* Reset the entire object. */
1087 gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND))); 1087 gcmkONERROR(gckOS_ZeroMemory(command, gcmSIZEOF(struct _gckCOMMAND)));
1088 1088
1089 /* Initialize the gckCOMMAND object.*/ 1089 /* Initialize the gckCOMMAND object.*/
1090 command->object.type = gcvOBJ_COMMAND; 1090 command->object.type = gcvOBJ_COMMAND;
1091 command->kernel = Kernel; 1091 command->kernel = Kernel;
1092 command->os = os; 1092 command->os = os;
1093 1093
1094 /* Get the command buffer requirements. */ 1094 /* Get the command buffer requirements. */
1095 gcmkONERROR(gckHARDWARE_QueryCommandBuffer( 1095 gcmkONERROR(gckHARDWARE_QueryCommandBuffer(
1096 Kernel->hardware, 1096 Kernel->hardware,
1097 gcvENGINE_RENDER, 1097 gcvENGINE_RENDER,
1098 &command->alignment, 1098 &command->alignment,
1099 &command->reservedHead, 1099 &command->reservedHead,
1100 gcvNULL 1100 gcvNULL
1101 )); 1101 ));
1102 1102
1103 /* Create the command queue mutex. */ 1103 /* Create the command queue mutex. */
1104 gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue)); 1104 gcmkONERROR(gckOS_CreateMutex(os, &command->mutexQueue));
1105 1105
1106 /* Create the context switching mutex. */ 1106 /* Create the context switching mutex. */
1107 gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext)); 1107 gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContext));
1108 1108
1109 /* Create the context switching mutex. */ 1109 /* Create the context switching mutex. */
1110 gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq)); 1110 gcmkONERROR(gckOS_CreateMutex(os, &command->mutexContextSeq));
1111 1111
1112 /* Create the power management semaphore. */ 1112 /* Create the power management semaphore. */
1113 gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore)); 1113 gcmkONERROR(gckOS_CreateSemaphore(os, &command->powerSemaphore));
1114 1114
1115 /* Create the commit atom. */ 1115 /* Create the commit atom. */
1116 gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit)); 1116 gcmkONERROR(gckOS_AtomConstruct(os, &command->atomCommit));
1117 1117
1118 /* Get the page size from teh OS. */ 1118 /* Get the page size from teh OS. */
1119 gcmkONERROR(gckOS_GetPageSize(os, &pageSize)); 1119 gcmkONERROR(gckOS_GetPageSize(os, &pageSize));
1120 1120
1121 gcmkSAFECASTSIZET(command->pageSize, pageSize); 1121 gcmkSAFECASTSIZET(command->pageSize, pageSize);
1122 1122
1123 /* Get process ID. */ 1123 /* Get process ID. */
1124 gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID)); 1124 gcmkONERROR(gckOS_GetProcessID(&command->kernelProcessID));
1125 1125
1126 /* Set hardware to pipe 0. */ 1126 /* Set hardware to pipe 0. */
1127 command->pipeSelect = gcvPIPE_INVALID; 1127 command->pipeSelect = gcvPIPE_INVALID;
1128 1128
1129 /* Pre-allocate the command queues. */ 1129 /* Pre-allocate the command queues. */
1130 for (i = 0; i < gcdCOMMAND_QUEUES; ++i) 1130 for (i = 0; i < gcdCOMMAND_QUEUES; ++i)
1131 { 1131 {
1132 #if USE_KERNEL_VIRTUAL_BUFFERS 1132 #if USE_KERNEL_VIRTUAL_BUFFERS
1133 if (Kernel->virtualCommandBuffer) 1133 if (Kernel->virtualCommandBuffer)
1134 { 1134 {
1135 gcmkONERROR(gckKERNEL_AllocateVirtualCommandBuffer( 1135 gcmkONERROR(gckKERNEL_AllocateVirtualCommandBuffer(
1136 Kernel, 1136 Kernel,
1137 gcvFALSE, 1137 gcvFALSE,
1138 &pageSize, 1138 &pageSize,
1139 &command->queues[i].physical, 1139 &command->queues[i].physical,
1140 &command->queues[i].logical 1140 &command->queues[i].logical
1141 )); 1141 ));
1142 1142
1143 gcmkONERROR(gckKERNEL_GetGPUAddress( 1143 gcmkONERROR(gckKERNEL_GetGPUAddress(
1144 Kernel, 1144 Kernel,
1145 command->queues[i].logical, 1145 command->queues[i].logical,
1146 gcvFALSE, 1146 gcvFALSE,
1147 command->queues[i].physical, 1147 command->queues[i].physical,
1148 &command->queues[i].address 1148 &command->queues[i].address
1149 )); 1149 ));
1150 } 1150 }
1151 else 1151 else
1152 #endif 1152 #endif
1153 { 1153 {
1154 gctUINT32 allocFlag; 1154 gctUINT32 allocFlag;
1155 1155
1156 #if gcdENABLE_CACHEABLE_COMMAND_BUFFER 1156 #if gcdENABLE_CACHEABLE_COMMAND_BUFFER
1157 allocFlag = gcvALLOC_FLAG_CACHEABLE | gcvALLOC_FLAG_CONTIGUOUS; 1157 allocFlag = gcvALLOC_FLAG_CACHEABLE | gcvALLOC_FLAG_CONTIGUOUS;
1158 #else 1158 #else
1159 allocFlag = gcvALLOC_FLAG_CONTIGUOUS; 1159 allocFlag = gcvALLOC_FLAG_CONTIGUOUS;
1160 #endif 1160 #endif
1161 1161
1162 gcmkONERROR(gckOS_AllocateNonPagedMemory( 1162 gcmkONERROR(gckOS_AllocateNonPagedMemory(
1163 os, 1163 os,
1164 gcvFALSE, 1164 gcvFALSE,
1165 allocFlag, 1165 allocFlag,
1166 &pageSize, 1166 &pageSize,
1167 &command->queues[i].physical, 1167 &command->queues[i].physical,
1168 &command->queues[i].logical 1168 &command->queues[i].logical
1169 )); 1169 ));
1170 1170
1171 gcmkONERROR(gckHARDWARE_ConvertLogical( 1171 gcmkONERROR(gckHARDWARE_ConvertLogical(
1172 Kernel->hardware, 1172 Kernel->hardware,
1173 command->queues[i].logical, 1173 command->queues[i].logical,
1174 gcvFALSE, 1174 gcvFALSE,
1175 &command->queues[i].address 1175 &command->queues[i].address
1176 )); 1176 ));
1177 1177
1178 gcmkONERROR(gckMMU_FillFlatMapping( 1178 gcmkONERROR(gckMMU_FillFlatMapping(
1179 Kernel->mmu, command->queues[i].address, pageSize 1179 Kernel->mmu, command->queues[i].address, pageSize
1180 )); 1180 ));
1181 } 1181 }
1182 1182
1183 gcmkONERROR(gckOS_CreateSignal( 1183 gcmkONERROR(gckOS_CreateSignal(
1184 os, gcvFALSE, &command->queues[i].signal 1184 os, gcvFALSE, &command->queues[i].signal
1185 )); 1185 ));
1186 1186
1187 gcmkONERROR(gckOS_Signal( 1187 gcmkONERROR(gckOS_Signal(
1188 os, command->queues[i].signal, gcvTRUE 1188 os, command->queues[i].signal, gcvTRUE
1189 )); 1189 ));
1190 } 1190 }
1191 1191
1192 #if gcdRECORD_COMMAND 1192 #if gcdRECORD_COMMAND
1193 gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder)); 1193 gcmkONERROR(gckRECORDER_Construct(os, Kernel->hardware, &command->recorder));
1194 #endif 1194 #endif
1195 1195
1196 gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsPATCH_LIST), &command->kList)); 1196 gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsPATCH_LIST), &command->kList));
1197 1197
1198 gcmkONERROR(gckFENCE_Create( 1198 gcmkONERROR(gckFENCE_Create(
1199 os, Kernel, &command->fence 1199 os, Kernel, &command->fence
1200 )); 1200 ));
1201 1201
1202 /* No command queue in use yet. */ 1202 /* No command queue in use yet. */
1203 command->index = -1; 1203 command->index = -1;
1204 command->logical = gcvNULL; 1204 command->logical = gcvNULL;
1205 command->newQueue = gcvFALSE; 1205 command->newQueue = gcvFALSE;
1206 1206
1207 /* Command is not yet running. */ 1207 /* Command is not yet running. */
1208 command->running = gcvFALSE; 1208 command->running = gcvFALSE;
1209 1209
1210 /* Command queue is idle. */ 1210 /* Command queue is idle. */
1211 command->idle = gcvTRUE; 1211 command->idle = gcvTRUE;
1212 1212
1213 /* Commit stamp start from 1. */ 1213 /* Commit stamp start from 1. */
1214 command->commitStamp = 1; 1214 command->commitStamp = 1;
1215 1215
1216 /* END event signal not created. */ 1216 /* END event signal not created. */
1217 command->endEventSignal = gcvNULL; 1217 command->endEventSignal = gcvNULL;
1218 1218
1219 command->dummyDraw = gcvTRUE; 1219 command->dummyDraw = gcvTRUE;
1220 1220
1221 /* Return pointer to the gckCOMMAND object. */ 1221 /* Return pointer to the gckCOMMAND object. */
1222 *Command = command; 1222 *Command = command;
1223 1223
1224 /* Success. */ 1224 /* Success. */
1225 gcmkFOOTER_ARG("*Command=0x%x", *Command); 1225 gcmkFOOTER_ARG("*Command=0x%x", *Command);
1226 return gcvSTATUS_OK; 1226 return gcvSTATUS_OK;
1227 1227
1228 OnError: 1228 OnError:
1229 /* Roll back. */ 1229 /* Roll back. */
1230 if (command != gcvNULL) 1230 if (command != gcvNULL)
1231 { 1231 {
1232 gcmkVERIFY_OK(gckCOMMAND_Destroy(command)); 1232 gcmkVERIFY_OK(gckCOMMAND_Destroy(command));
1233 } 1233 }
1234 1234
1235 /* Return the status. */ 1235 /* Return the status. */
1236 gcmkFOOTER(); 1236 gcmkFOOTER();
1237 return status; 1237 return status;
1238 } 1238 }
1239 1239
1240 /******************************************************************************* 1240 /*******************************************************************************
1241 ** 1241 **
1242 ** gckCOMMAND_Destroy 1242 ** gckCOMMAND_Destroy
1243 ** 1243 **
1244 ** Destroy an gckCOMMAND object. 1244 ** Destroy an gckCOMMAND object.
1245 ** 1245 **
1246 ** INPUT: 1246 ** INPUT:
1247 ** 1247 **
1248 ** gckCOMMAND Command 1248 ** gckCOMMAND Command
1249 ** Pointer to an gckCOMMAND object to destroy. 1249 ** Pointer to an gckCOMMAND object to destroy.
1250 ** 1250 **
1251 ** OUTPUT: 1251 ** OUTPUT:
1252 ** 1252 **
1253 ** Nothing. 1253 ** Nothing.
1254 */ 1254 */
1255 gceSTATUS 1255 gceSTATUS
1256 gckCOMMAND_Destroy( 1256 gckCOMMAND_Destroy(
1257 IN gckCOMMAND Command 1257 IN gckCOMMAND Command
1258 ) 1258 )
1259 { 1259 {
1260 gctINT i; 1260 gctINT i;
1261 1261
1262 gcmkHEADER_ARG("Command=0x%x", Command); 1262 gcmkHEADER_ARG("Command=0x%x", Command);
1263 1263
1264 /* Verify the arguments. */ 1264 /* Verify the arguments. */
1265 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 1265 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
1266 1266
1267 /* Stop the command queue. */ 1267 /* Stop the command queue. */
1268 gcmkVERIFY_OK(gckCOMMAND_Stop(Command)); 1268 gcmkVERIFY_OK(gckCOMMAND_Stop(Command));
1269 1269
1270 for (i = 0; i < gcdCOMMAND_QUEUES; ++i) 1270 for (i = 0; i < gcdCOMMAND_QUEUES; ++i)
1271 { 1271 {
1272 if (Command->queues[i].signal) 1272 if (Command->queues[i].signal)
1273 { 1273 {
1274 gcmkVERIFY_OK(gckOS_DestroySignal( 1274 gcmkVERIFY_OK(gckOS_DestroySignal(
1275 Command->os, Command->queues[i].signal 1275 Command->os, Command->queues[i].signal
1276 )); 1276 ));
1277 } 1277 }
1278 1278
1279 if (Command->queues[i].logical) 1279 if (Command->queues[i].logical)
1280 { 1280 {
1281 #if USE_KERNEL_VIRTUAL_BUFFERS 1281 #if USE_KERNEL_VIRTUAL_BUFFERS
1282 if (Command->kernel->virtualCommandBuffer) 1282 if (Command->kernel->virtualCommandBuffer)
1283 { 1283 {
1284 gcmkVERIFY_OK(gckKERNEL_DestroyVirtualCommandBuffer( 1284 gcmkVERIFY_OK(gckKERNEL_DestroyVirtualCommandBuffer(
1285 Command->kernel, 1285 Command->kernel,
1286 Command->pageSize, 1286 Command->pageSize,
1287 Command->queues[i].physical, 1287 Command->queues[i].physical,
1288 Command->queues[i].logical 1288 Command->queues[i].logical
1289 )); 1289 ));
1290 } 1290 }
1291 else 1291 else
1292 #endif 1292 #endif
1293 { 1293 {
1294 gcmkVERIFY_OK(gckOS_FreeNonPagedMemory( 1294 gcmkVERIFY_OK(gckOS_FreeNonPagedMemory(
1295 Command->os, 1295 Command->os,
1296 Command->pageSize, 1296 Command->pageSize,
1297 Command->queues[i].physical, 1297 Command->queues[i].physical,
1298 Command->queues[i].logical 1298 Command->queues[i].logical
1299 )); 1299 ));
1300 } 1300 }
1301 } 1301 }
1302 } 1302 }
1303 1303
1304 /* END event signal. */ 1304 /* END event signal. */
1305 if (Command->endEventSignal != gcvNULL) 1305 if (Command->endEventSignal != gcvNULL)
1306 { 1306 {
1307 gcmkVERIFY_OK(gckOS_DestroySignal( 1307 gcmkVERIFY_OK(gckOS_DestroySignal(
1308 Command->os, Command->endEventSignal 1308 Command->os, Command->endEventSignal
1309 )); 1309 ));
1310 } 1310 }
1311 1311
1312 if (Command->mutexContext) 1312 if (Command->mutexContext)
1313 { 1313 {
1314 /* Delete the context switching mutex. */ 1314 /* Delete the context switching mutex. */
1315 gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext)); 1315 gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContext));
1316 } 1316 }
1317 1317
1318 if (Command->mutexContextSeq != gcvNULL) 1318 if (Command->mutexContextSeq != gcvNULL)
1319 gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq)); 1319 gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexContextSeq));
1320 1320
1321 if (Command->mutexQueue) 1321 if (Command->mutexQueue)
1322 { 1322 {
1323 /* Delete the command queue mutex. */ 1323 /* Delete the command queue mutex. */
1324 gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue)); 1324 gcmkVERIFY_OK(gckOS_DeleteMutex(Command->os, Command->mutexQueue));
1325 } 1325 }
1326 1326
1327 if (Command->powerSemaphore) 1327 if (Command->powerSemaphore)
1328 { 1328 {
1329 /* Destroy the power management semaphore. */ 1329 /* Destroy the power management semaphore. */
1330 gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore)); 1330 gcmkVERIFY_OK(gckOS_DestroySemaphore(Command->os, Command->powerSemaphore));
1331 } 1331 }
1332 1332
1333 if (Command->atomCommit) 1333 if (Command->atomCommit)
1334 { 1334 {
1335 /* Destroy the commit atom. */ 1335 /* Destroy the commit atom. */
1336 gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit)); 1336 gcmkVERIFY_OK(gckOS_AtomDestroy(Command->os, Command->atomCommit));
1337 } 1337 }
1338 1338
1339 #if gcdSECURE_USER 1339 #if gcdSECURE_USER
1340 /* Free state array. */ 1340 /* Free state array. */
1341 if (Command->hintArrayAllocated) 1341 if (Command->hintArrayAllocated)
1342 { 1342 {
1343 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray))); 1343 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, gcmUINT64_TO_PTR(Command->hintArray)));
1344 Command->hintArrayAllocated = gcvFALSE; 1344 Command->hintArrayAllocated = gcvFALSE;
1345 } 1345 }
1346 #endif 1346 #endif
1347 1347
1348 #if gcdRECORD_COMMAND 1348 #if gcdRECORD_COMMAND
1349 gckRECORDER_Destory(Command->os, Command->recorder); 1349 gckRECORDER_Destory(Command->os, Command->recorder);
1350 #endif 1350 #endif
1351 1351
1352 if (Command->stateMap) 1352 if (Command->stateMap)
1353 { 1353 {
1354 gcmkOS_SAFE_FREE(Command->os, Command->stateMap); 1354 gcmkOS_SAFE_FREE(Command->os, Command->stateMap);
1355 } 1355 }
1356 1356
1357 if (Command->kList) 1357 if (Command->kList)
1358 { 1358 {
1359 gcmkOS_SAFE_FREE(Command->os, Command->kList); 1359 gcmkOS_SAFE_FREE(Command->os, Command->kList);
1360 } 1360 }
1361 1361
1362 if (Command->fence) 1362 if (Command->fence)
1363 { 1363 {
1364 gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence)); 1364 gcmkVERIFY_OK(gckFENCE_Destory(Command->os, Command->fence));
1365 } 1365 }
1366 1366
1367 /* Mark object as unknown. */ 1367 /* Mark object as unknown. */
1368 Command->object.type = gcvOBJ_UNKNOWN; 1368 Command->object.type = gcvOBJ_UNKNOWN;
1369 1369
1370 /* Free the gckCOMMAND object. */ 1370 /* Free the gckCOMMAND object. */
1371 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command)); 1371 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Command->os, Command));
1372 1372
1373 /* Success. */ 1373 /* Success. */
1374 gcmkFOOTER_NO(); 1374 gcmkFOOTER_NO();
1375 return gcvSTATUS_OK; 1375 return gcvSTATUS_OK;
1376 } 1376 }
1377 1377
1378 /******************************************************************************* 1378 /*******************************************************************************
1379 ** 1379 **
1380 ** gckCOMMAND_EnterCommit 1380 ** gckCOMMAND_EnterCommit
1381 ** 1381 **
1382 ** Acquire command queue synchronization objects. 1382 ** Acquire command queue synchronization objects.
1383 ** 1383 **
1384 ** INPUT: 1384 ** INPUT:
1385 ** 1385 **
1386 ** gckCOMMAND Command 1386 ** gckCOMMAND Command
1387 ** Pointer to an gckCOMMAND object to destroy. 1387 ** Pointer to an gckCOMMAND object to destroy.
1388 ** 1388 **
1389 ** gctBOOL FromPower 1389 ** gctBOOL FromPower
1390 ** Determines whether the call originates from inside the power 1390 ** Determines whether the call originates from inside the power
1391 ** management or not. 1391 ** management or not.
1392 ** 1392 **
1393 ** OUTPUT: 1393 ** OUTPUT:
1394 ** 1394 **
1395 ** Nothing. 1395 ** Nothing.
1396 */ 1396 */
1397 gceSTATUS 1397 gceSTATUS
1398 gckCOMMAND_EnterCommit( 1398 gckCOMMAND_EnterCommit(
1399 IN gckCOMMAND Command, 1399 IN gckCOMMAND Command,
1400 IN gctBOOL FromPower 1400 IN gctBOOL FromPower
1401 ) 1401 )
1402 { 1402 {
1403 gceSTATUS status; 1403 gceSTATUS status;
1404 gckHARDWARE hardware; 1404 gckHARDWARE hardware;
1405 gctBOOL atomIncremented = gcvFALSE; 1405 gctBOOL atomIncremented = gcvFALSE;
1406 gctBOOL semaAcquired = gcvFALSE; 1406 gctBOOL semaAcquired = gcvFALSE;
1407 1407
1408 gcmkHEADER_ARG("Command=0x%x", Command); 1408 gcmkHEADER_ARG("Command=0x%x", Command);
1409 1409
1410 /* Extract the gckHARDWARE and gckEVENT objects. */ 1410 /* Extract the gckHARDWARE and gckEVENT objects. */
1411 hardware = Command->kernel->hardware; 1411 hardware = Command->kernel->hardware;
1412 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); 1412 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
1413 1413
1414 if (!FromPower) 1414 if (!FromPower)
1415 { 1415 {
1416 /* Increment COMMIT atom to let power management know that a commit is 1416 /* Increment COMMIT atom to let power management know that a commit is
1417 ** in progress. */ 1417 ** in progress. */
1418 gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE)); 1418 gcmkONERROR(_IncrementCommitAtom(Command, gcvTRUE));
1419 atomIncremented = gcvTRUE; 1419 atomIncremented = gcvTRUE;
1420 1420
1421 /* Notify the system the GPU has a commit. */ 1421 /* Notify the system the GPU has a commit. */
1422 gcmkONERROR(gckOS_Broadcast(Command->os, 1422 gcmkONERROR(gckOS_Broadcast(Command->os,
1423 hardware, 1423 hardware,
1424 gcvBROADCAST_GPU_COMMIT)); 1424 gcvBROADCAST_GPU_COMMIT));
1425 1425
1426 /* Acquire the power management semaphore. */ 1426 /* Acquire the power management semaphore. */
1427 gcmkONERROR(gckOS_AcquireSemaphore(Command->os, 1427 gcmkONERROR(gckOS_AcquireSemaphore(Command->os,
1428 Command->powerSemaphore)); 1428 Command->powerSemaphore));
1429 semaAcquired = gcvTRUE; 1429 semaAcquired = gcvTRUE;
1430 } 1430 }
1431 1431
1432 /* Grab the conmmand queue mutex. */ 1432 /* Grab the conmmand queue mutex. */
1433 gcmkONERROR(gckOS_AcquireMutex(Command->os, 1433 gcmkONERROR(gckOS_AcquireMutex(Command->os,
1434 Command->mutexQueue, 1434 Command->mutexQueue,
1435 gcvINFINITE)); 1435 gcvINFINITE));
1436 1436
1437 /* Success. */ 1437 /* Success. */
1438 gcmkFOOTER(); 1438 gcmkFOOTER();
1439 return gcvSTATUS_OK; 1439 return gcvSTATUS_OK;
1440 1440
1441 OnError: 1441 OnError:
1442 if (semaAcquired) 1442 if (semaAcquired)
1443 { 1443 {
1444 /* Release the power management semaphore. */ 1444 /* Release the power management semaphore. */
1445 gcmkVERIFY_OK(gckOS_ReleaseSemaphore( 1445 gcmkVERIFY_OK(gckOS_ReleaseSemaphore(
1446 Command->os, Command->powerSemaphore 1446 Command->os, Command->powerSemaphore
1447 )); 1447 ));
1448 } 1448 }
1449 1449
1450 if (atomIncremented) 1450 if (atomIncremented)
1451 { 1451 {
1452 /* Decrement the commit atom. */ 1452 /* Decrement the commit atom. */
1453 gcmkVERIFY_OK(_IncrementCommitAtom( 1453 gcmkVERIFY_OK(_IncrementCommitAtom(
1454 Command, gcvFALSE 1454 Command, gcvFALSE
1455 )); 1455 ));
1456 } 1456 }
1457 1457
1458 /* Return the status. */ 1458 /* Return the status. */
1459 gcmkFOOTER(); 1459 gcmkFOOTER();
1460 return status; 1460 return status;
1461 } 1461 }
1462 1462
1463 /******************************************************************************* 1463 /*******************************************************************************
1464 ** 1464 **
1465 ** gckCOMMAND_ExitCommit 1465 ** gckCOMMAND_ExitCommit
1466 ** 1466 **
1467 ** Release command queue synchronization objects. 1467 ** Release command queue synchronization objects.
1468 ** 1468 **
1469 ** INPUT: 1469 ** INPUT:
1470 ** 1470 **
1471 ** gckCOMMAND Command 1471 ** gckCOMMAND Command
1472 ** Pointer to an gckCOMMAND object to destroy. 1472 ** Pointer to an gckCOMMAND object to destroy.
1473 ** 1473 **
1474 ** gctBOOL FromPower 1474 ** gctBOOL FromPower
1475 ** Determines whether the call originates from inside the power 1475 ** Determines whether the call originates from inside the power
1476 ** management or not. 1476 ** management or not.
1477 ** 1477 **
1478 ** OUTPUT: 1478 ** OUTPUT:
1479 ** 1479 **
1480 ** Nothing. 1480 ** Nothing.
1481 */ 1481 */
1482 gceSTATUS 1482 gceSTATUS
1483 gckCOMMAND_ExitCommit( 1483 gckCOMMAND_ExitCommit(
1484 IN gckCOMMAND Command, 1484 IN gckCOMMAND Command,
1485 IN gctBOOL FromPower 1485 IN gctBOOL FromPower
1486 ) 1486 )
1487 { 1487 {
1488 gceSTATUS status; 1488 gceSTATUS status;
1489 1489
1490 gcmkHEADER_ARG("Command=0x%x", Command); 1490 gcmkHEADER_ARG("Command=0x%x", Command);
1491 1491
1492 /* Release the power mutex. */ 1492 /* Release the power mutex. */
1493 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue)); 1493 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexQueue));
1494 1494
1495 if (!FromPower) 1495 if (!FromPower)
1496 { 1496 {
1497 /* Release the power management semaphore. */ 1497 /* Release the power management semaphore. */
1498 gcmkONERROR(gckOS_ReleaseSemaphore(Command->os, 1498 gcmkONERROR(gckOS_ReleaseSemaphore(Command->os,
1499 Command->powerSemaphore)); 1499 Command->powerSemaphore));
1500 1500
1501 /* Decrement the commit atom. */ 1501 /* Decrement the commit atom. */
1502 gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE)); 1502 gcmkONERROR(_IncrementCommitAtom(Command, gcvFALSE));
1503 } 1503 }
1504 1504
1505 /* Success. */ 1505 /* Success. */
1506 gcmkFOOTER(); 1506 gcmkFOOTER();
1507 return gcvSTATUS_OK; 1507 return gcvSTATUS_OK;
1508 1508
1509 OnError: 1509 OnError:
1510 /* Return the status. */ 1510 /* Return the status. */
1511 gcmkFOOTER(); 1511 gcmkFOOTER();
1512 return status; 1512 return status;
1513 } 1513 }
1514 1514
1515 /******************************************************************************* 1515 /*******************************************************************************
1516 ** 1516 **
1517 ** gckCOMMAND_Start 1517 ** gckCOMMAND_Start
1518 ** 1518 **
1519 ** Start up the command queue. 1519 ** Start up the command queue.
1520 ** 1520 **
1521 ** INPUT: 1521 ** INPUT:
1522 ** 1522 **
1523 ** gckCOMMAND Command 1523 ** gckCOMMAND Command
1524 ** Pointer to an gckCOMMAND object to start. 1524 ** Pointer to an gckCOMMAND object to start.
1525 ** 1525 **
1526 ** OUTPUT: 1526 ** OUTPUT:
1527 ** 1527 **
1528 ** Nothing. 1528 ** Nothing.
1529 */ 1529 */
1530 gceSTATUS 1530 gceSTATUS
1531 gckCOMMAND_Start( 1531 gckCOMMAND_Start(
1532 IN gckCOMMAND Command 1532 IN gckCOMMAND Command
1533 ) 1533 )
1534 { 1534 {
1535 gceSTATUS status; 1535 gceSTATUS status;
1536 gckHARDWARE hardware; 1536 gckHARDWARE hardware;
1537 gctUINT32 waitOffset = 0; 1537 gctUINT32 waitOffset = 0;
1538 gctUINT32 waitLinkBytes; 1538 gctUINT32 waitLinkBytes;
1539 gctPOINTER logical; 1539 gctPOINTER logical;
1540 gctUINT32 physical; 1540 gctUINT32 physical;
1541 gctUINT32 address; 1541 gctUINT32 address;
1542 1542
1543 gcmkHEADER_ARG("Command=0x%x", Command); 1543 gcmkHEADER_ARG("Command=0x%x", Command);
1544 1544
1545 /* Verify the arguments. */ 1545 /* Verify the arguments. */
1546 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 1546 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
1547 1547
1548 if (Command->running) 1548 if (Command->running)
1549 { 1549 {
1550 /* Command queue already running. */ 1550 /* Command queue already running. */
1551 gcmkFOOTER_NO(); 1551 gcmkFOOTER_NO();
1552 return gcvSTATUS_OK; 1552 return gcvSTATUS_OK;
1553 } 1553 }
1554 1554
1555 /* Extract the gckHARDWARE object. */ 1555 /* Extract the gckHARDWARE object. */
1556 hardware = Command->kernel->hardware; 1556 hardware = Command->kernel->hardware;
1557 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); 1557 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
1558 1558
1559 /* Query the size of WAIT/LINK command sequence. */ 1559 /* Query the size of WAIT/LINK command sequence. */
1560 gcmkONERROR(gckHARDWARE_WaitLink( 1560 gcmkONERROR(gckHARDWARE_WaitLink(
1561 hardware, 1561 hardware,
1562 gcvNULL, 1562 gcvNULL,
1563 ~0U, 1563 ~0U,
1564 Command->offset, 1564 Command->offset,
1565 &waitLinkBytes, 1565 &waitLinkBytes,
1566 gcvNULL, 1566 gcvNULL,
1567 gcvNULL 1567 gcvNULL
1568 )); 1568 ));
1569 1569
1570 if ((Command->pageSize - Command->offset < waitLinkBytes) 1570 if ((Command->pageSize - Command->offset < waitLinkBytes)
1571 || (Command->logical == gcvNULL) 1571 || (Command->logical == gcvNULL)
1572 ) 1572 )
1573 { 1573 {
1574 /* Start at beginning of a new queue. */ 1574 /* Start at beginning of a new queue. */
1575 gcmkONERROR(_NewQueue(Command, gcvTRUE)); 1575 gcmkONERROR(_NewQueue(Command, gcvTRUE));
1576 } 1576 }
1577 1577
1578 logical = (gctUINT8_PTR) Command->logical + Command->offset; 1578 logical = (gctUINT8_PTR) Command->logical + Command->offset;
1579 physical = Command->physical + Command->offset; 1579 physical = Command->physical + Command->offset;
1580 address = Command->address + Command->offset; 1580 address = Command->address + Command->offset;
1581 1581
1582 /* Append WAIT/LINK. */ 1582 /* Append WAIT/LINK. */
1583 gcmkONERROR(gckHARDWARE_WaitLink( 1583 gcmkONERROR(gckHARDWARE_WaitLink(
1584 hardware, 1584 hardware,
1585 logical, 1585 logical,
1586 address, 1586 address,
1587 0, 1587 0,
1588 &waitLinkBytes, 1588 &waitLinkBytes,
1589 &waitOffset, 1589 &waitOffset,
1590 &Command->waitSize 1590 &Command->waitSize
1591 )); 1591 ));
1592 1592
1593 Command->waitLogical = (gctUINT8_PTR) logical + waitOffset; 1593 Command->waitLogical = (gctUINT8_PTR) logical + waitOffset;
1594 Command->waitPhysical = physical + waitOffset; 1594 Command->waitPhysical = physical + waitOffset;
1595 Command->waitAddress = address + waitOffset; 1595 Command->waitAddress = address + waitOffset;
1596 Command->waitOffset = waitOffset; 1596 Command->waitOffset = waitOffset;
1597 1597
1598 /* Flush the cache for the wait/link. */ 1598 /* Flush the cache for the wait/link. */
1599 gcmkONERROR(gckOS_CacheClean( 1599 gcmkONERROR(gckOS_CacheClean(
1600 Command->os, 1600 Command->os,
1601 0, 1601 0,
1602 Command->physHandle, 1602 Command->physHandle,
1603 Command->offset, 1603 Command->offset,
1604 logical, 1604 logical,
1605 waitLinkBytes 1605 waitLinkBytes
1606 )); 1606 ));
1607 1607
1608 /* Adjust offset. */ 1608 /* Adjust offset. */
1609 Command->offset += waitLinkBytes; 1609 Command->offset += waitLinkBytes;
1610 Command->newQueue = gcvFALSE; 1610 Command->newQueue = gcvFALSE;
1611 1611
1612 #if gcdSECURITY 1612 #if gcdSECURITY
1613 /* Start FE by calling security service. */ 1613 /* Start FE by calling security service. */
1614 gckKERNEL_SecurityStartCommand( 1614 gckKERNEL_SecurityStartCommand(
1615 Command->kernel 1615 Command->kernel
1616 ); 1616 );
1617 #else 1617 #else
1618 /* Enable command processor. */ 1618 /* Enable command processor. */
1619 gcmkONERROR(gckHARDWARE_Execute( 1619 gcmkONERROR(gckHARDWARE_Execute(
1620 hardware, 1620 hardware,
1621 address, 1621 address,
1622 waitLinkBytes 1622 waitLinkBytes
1623 )); 1623 ));
1624 #endif 1624 #endif
1625 1625
1626 /* Command queue is running. */ 1626 /* Command queue is running. */
1627 Command->running = gcvTRUE; 1627 Command->running = gcvTRUE;
1628 1628
1629 /* Success. */ 1629 /* Success. */
1630 gcmkFOOTER_NO(); 1630 gcmkFOOTER_NO();
1631 return gcvSTATUS_OK; 1631 return gcvSTATUS_OK;
1632 1632
1633 OnError: 1633 OnError:
1634 /* Return the status. */ 1634 /* Return the status. */
1635 gcmkFOOTER(); 1635 gcmkFOOTER();
1636 return status; 1636 return status;
1637 } 1637 }
1638 1638
1639 /******************************************************************************* 1639 /*******************************************************************************
1640 ** 1640 **
1641 ** gckCOMMAND_Stop 1641 ** gckCOMMAND_Stop
1642 ** 1642 **
1643 ** Stop the command queue. 1643 ** Stop the command queue.
1644 ** 1644 **
1645 ** INPUT: 1645 ** INPUT:
1646 ** 1646 **
1647 ** gckCOMMAND Command 1647 ** gckCOMMAND Command
1648 ** Pointer to an gckCOMMAND object to stop. 1648 ** Pointer to an gckCOMMAND object to stop.
1649 ** 1649 **
1650 ** OUTPUT: 1650 ** OUTPUT:
1651 ** 1651 **
1652 ** Nothing. 1652 ** Nothing.
1653 */ 1653 */
1654 gceSTATUS 1654 gceSTATUS
1655 gckCOMMAND_Stop( 1655 gckCOMMAND_Stop(
1656 IN gckCOMMAND Command 1656 IN gckCOMMAND Command
1657 ) 1657 )
1658 { 1658 {
1659 gckHARDWARE hardware; 1659 gckHARDWARE hardware;
1660 gceSTATUS status; 1660 gceSTATUS status;
1661 gctUINT32 idle; 1661 gctUINT32 idle;
1662 1662
1663 gcmkHEADER_ARG("Command=0x%x", Command); 1663 gcmkHEADER_ARG("Command=0x%x", Command);
1664 1664
1665 /* Verify the arguments. */ 1665 /* Verify the arguments. */
1666 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 1666 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
1667 1667
1668 if (!Command->running) 1668 if (!Command->running)
1669 { 1669 {
1670 /* Command queue is not running. */ 1670 /* Command queue is not running. */
1671 gcmkFOOTER_NO(); 1671 gcmkFOOTER_NO();
1672 return gcvSTATUS_OK; 1672 return gcvSTATUS_OK;
1673 } 1673 }
1674 1674
1675 /* Extract the gckHARDWARE object. */ 1675 /* Extract the gckHARDWARE object. */
1676 hardware = Command->kernel->hardware; 1676 hardware = Command->kernel->hardware;
1677 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); 1677 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
1678 1678
1679 if (gckHARDWARE_IsFeatureAvailable(hardware, 1679 if (gckHARDWARE_IsFeatureAvailable(hardware,
1680 gcvFEATURE_END_EVENT) == gcvSTATUS_TRUE) 1680 gcvFEATURE_END_EVENT) == gcvSTATUS_TRUE)
1681 { 1681 {
1682 /* Allocate the signal. */ 1682 /* Allocate the signal. */
1683 if (Command->endEventSignal == gcvNULL) 1683 if (Command->endEventSignal == gcvNULL)
1684 { 1684 {
1685 gcmkONERROR(gckOS_CreateSignal(Command->os, 1685 gcmkONERROR(gckOS_CreateSignal(Command->os,
1686 gcvTRUE, 1686 gcvTRUE,
1687 &Command->endEventSignal)); 1687 &Command->endEventSignal));
1688 } 1688 }
1689 1689
1690 /* Append the END EVENT command to trigger the signal. */ 1690 /* Append the END EVENT command to trigger the signal. */
1691 gcmkONERROR(gckEVENT_Stop(Command->kernel->eventObj, 1691 gcmkONERROR(gckEVENT_Stop(Command->kernel->eventObj,
1692 Command->kernelProcessID, 1692 Command->kernelProcessID,
1693 Command->physHandle, 1693 Command->physHandle,
1694 Command->offset + Command->waitOffset, 1694 Command->offset + Command->waitOffset,
1695 Command->waitLogical, 1695 Command->waitLogical,
1696 Command->waitAddress, 1696 Command->waitAddress,
1697 Command->endEventSignal, 1697 Command->endEventSignal,
1698 &Command->waitSize)); 1698 &Command->waitSize));
1699 } 1699 }
1700 else 1700 else
1701 { 1701 {
1702 /* Replace last WAIT with END. */ 1702 /* Replace last WAIT with END. */
1703 gcmkONERROR(gckHARDWARE_End( 1703 gcmkONERROR(gckHARDWARE_End(
1704 hardware, 1704 hardware,
1705 Command->waitLogical, 1705 Command->waitLogical,
1706 Command->waitAddress, 1706 Command->waitAddress,
1707 &Command->waitSize 1707 &Command->waitSize
1708 )); 1708 ));
1709 1709
1710 #if USE_KERNEL_VIRTUAL_BUFFERS 1710 #if USE_KERNEL_VIRTUAL_BUFFERS
1711 if (hardware->kernel->virtualCommandBuffer) 1711 if (hardware->kernel->virtualCommandBuffer)
1712 { 1712 {
1713 gcmkONERROR(gckKERNEL_GetGPUAddress( 1713 gcmkONERROR(gckKERNEL_GetGPUAddress(
1714 hardware->kernel, 1714 hardware->kernel,
1715 Command->waitLogical, 1715 Command->waitLogical,
1716 gcvFALSE, 1716 gcvFALSE,
1717 Command->virtualMemory, 1717 Command->virtualMemory,
1718 &hardware->lastEnd 1718 &hardware->lastEnd
1719 )); 1719 ));
1720 } 1720 }
1721 #endif 1721 #endif
1722 1722
1723 #if gcdSECURITY 1723 #if gcdSECURITY
1724 gcmkONERROR(gckKERNEL_SecurityExecute( 1724 gcmkONERROR(gckKERNEL_SecurityExecute(
1725 Command->kernel, Command->waitLogical, 8 1725 Command->kernel, Command->waitLogical, 8
1726 )); 1726 ));
1727 #endif 1727 #endif
1728 1728
1729 /* Update queue tail pointer. */ 1729 /* Update queue tail pointer. */
1730 gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware, 1730 gcmkONERROR(gckHARDWARE_UpdateQueueTail(Command->kernel->hardware,
1731 Command->logical, 1731 Command->logical,
1732 Command->offset)); 1732 Command->offset));
1733 1733
1734 /* Flush the cache for the END. */ 1734 /* Flush the cache for the END. */
1735 gcmkONERROR(gckOS_CacheClean( 1735 gcmkONERROR(gckOS_CacheClean(
1736 Command->os, 1736 Command->os,
1737 0, 1737 0,
1738 Command->physHandle, 1738 Command->physHandle,
1739 Command->offset + Command->waitOffset, 1739 Command->offset + Command->waitOffset,
1740 Command->waitLogical, 1740 Command->waitLogical,
1741 Command->waitSize 1741 Command->waitSize
1742 )); 1742 ));
1743 1743
1744 /* Wait for idle. */ 1744 /* Wait for idle. */
1745 gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle)); 1745 gcmkONERROR(gckHARDWARE_GetIdle(hardware, gcvTRUE, &idle));
1746 } 1746 }
1747 1747
1748 /* Command queue is no longer running. */ 1748 /* Command queue is no longer running. */
1749 Command->running = gcvFALSE; 1749 Command->running = gcvFALSE;
1750 1750
1751 /* Success. */ 1751 /* Success. */
1752 gcmkFOOTER_NO(); 1752 gcmkFOOTER_NO();
1753 return gcvSTATUS_OK; 1753 return gcvSTATUS_OK;
1754 1754
1755 OnError: 1755 OnError:
1756 /* Return the status. */ 1756 /* Return the status. */
1757 gcmkFOOTER(); 1757 gcmkFOOTER();
1758 return status; 1758 return status;
1759 } 1759 }
1760 1760
1761 /******************************************************************************* 1761 /*******************************************************************************
1762 ** 1762 **
1763 ** gckCOMMAND_Commit 1763 ** gckCOMMAND_Commit
1764 ** 1764 **
1765 ** Commit a command buffer to the command queue. 1765 ** Commit a command buffer to the command queue.
1766 ** 1766 **
1767 ** INPUT: 1767 ** INPUT:
1768 ** 1768 **
1769 ** gckCOMMAND Command 1769 ** gckCOMMAND Command
1770 ** Pointer to a gckCOMMAND object. 1770 ** Pointer to a gckCOMMAND object.
1771 ** 1771 **
1772 ** gckCONTEXT Context 1772 ** gckCONTEXT Context
1773 ** Pointer to a gckCONTEXT object. 1773 ** Pointer to a gckCONTEXT object.
1774 ** 1774 **
1775 ** gcoCMDBUF CommandBuffer 1775 ** gcoCMDBUF CommandBuffer
1776 ** Pointer to a gcoCMDBUF object. 1776 ** Pointer to a gcoCMDBUF object.
1777 ** 1777 **
1778 ** gcsSTATE_DELTA_PTR StateDelta 1778 ** gcsSTATE_DELTA_PTR StateDelta
1779 ** Pointer to the state delta. 1779 ** Pointer to the state delta.
1780 ** 1780 **
1781 ** gctUINT32 ProcessID 1781 ** gctUINT32 ProcessID
1782 ** Current process ID. 1782 ** Current process ID.
1783 ** 1783 **
1784 ** OUTPUT: 1784 ** OUTPUT:
1785 ** 1785 **
1786 ** Nothing. 1786 ** Nothing.
1787 */ 1787 */
1788 gceSTATUS 1788 gceSTATUS
1789 gckCOMMAND_Commit( 1789 gckCOMMAND_Commit(
1790 IN gckCOMMAND Command, 1790 IN gckCOMMAND Command,
1791 IN gckCONTEXT Context, 1791 IN gckCONTEXT Context,
1792 IN gcoCMDBUF CommandBuffer, 1792 IN gcoCMDBUF CommandBuffer,
1793 IN gcsSTATE_DELTA_PTR StateDelta, 1793 IN gcsSTATE_DELTA_PTR StateDelta,
1794 IN gctUINT32 ProcessID, 1794 IN gctUINT32 ProcessID,
1795 IN gctBOOL Shared, 1795 IN gctBOOL Shared,
1796 IN gctUINT32 Index, 1796 IN gctUINT32 Index,
1797 OUT gctUINT64_PTR CommitStamp, 1797 OUT gctUINT64_PTR CommitStamp,
1798 OUT gctBOOL_PTR ContextSwitched 1798 OUT gctBOOL_PTR ContextSwitched
1799 ) 1799 )
1800 { 1800 {
1801 gceSTATUS status; 1801 gceSTATUS status;
1802 gctBOOL commitEntered = gcvFALSE; 1802 gctBOOL commitEntered = gcvFALSE;
1803 gctBOOL contextAcquired = gcvFALSE; 1803 gctBOOL contextAcquired = gcvFALSE;
1804 gckHARDWARE hardware; 1804 gckHARDWARE hardware;
1805 gctBOOL needCopy = gcvFALSE; 1805 gctBOOL needCopy = gcvFALSE;
1806 gctBOOL commandBufferMapped = gcvFALSE; 1806 gctBOOL commandBufferMapped = gcvFALSE;
1807 gcoCMDBUF commandBufferObject = gcvNULL; 1807 gcoCMDBUF commandBufferObject = gcvNULL;
1808 gctBOOL stall = gcvFALSE; 1808 gctBOOL stall = gcvFALSE;
1809 gctBOOL contextSwitched = gcvFALSE; 1809 gctBOOL contextSwitched = gcvFALSE;
1810 1810
1811 #if !gcdNULL_DRIVER 1811 #if !gcdNULL_DRIVER
1812 gcsCONTEXT_PTR contextBuffer; 1812 gcsCONTEXT_PTR contextBuffer;
1813 gctPHYS_ADDR_T commandBufferPhysical; 1813 gctPHYS_ADDR_T commandBufferPhysical;
1814 gctUINT8_PTR commandBufferLogical = gcvNULL; 1814 gctUINT8_PTR commandBufferLogical = gcvNULL;
1815 gctUINT32 commandBufferAddress = 0; 1815 gctUINT32 commandBufferAddress = 0;
1816 gctUINT8_PTR commandBufferTail = gcvNULL; 1816 gctUINT8_PTR commandBufferTail = gcvNULL;
1817 gctUINT commandBufferSize; 1817 gctUINT commandBufferSize;
1818 gctSIZE_T nopBytes; 1818 gctSIZE_T nopBytes;
1819 gctUINT32 pipeBytes; 1819 gctUINT32 pipeBytes;
1820 gctUINT32 linkBytes; 1820 gctUINT32 linkBytes;
1821 gctSIZE_T bytes; 1821 gctSIZE_T bytes;
1822 gctUINT32 offset; 1822 gctUINT32 offset;
1823 gctPOINTER entryLogical; 1823 gctPOINTER entryLogical;
1824 gctUINT32 entryAddress; 1824 gctUINT32 entryAddress;
1825 gctUINT32 entryBytes; 1825 gctUINT32 entryBytes;
1826 gctUINT32 exitOffset; 1826 gctUINT32 exitOffset;
1827 gctPOINTER exitLogical; 1827 gctPOINTER exitLogical;
1828 gctUINT32 exitAddress; 1828 gctUINT32 exitAddress;
1829 gctUINT32 exitBytes; 1829 gctUINT32 exitBytes;
1830 gctUINT32 waitLinkPhysical; 1830 gctUINT32 waitLinkPhysical;
1831 gctPOINTER waitLinkLogical; 1831 gctPOINTER waitLinkLogical;
1832 gctUINT32 waitLinkAddress; 1832 gctUINT32 waitLinkAddress;
1833 gctUINT32 waitLinkBytes; 1833 gctUINT32 waitLinkBytes;
1834 gctUINT32 waitOffset; 1834 gctUINT32 waitOffset;
1835 gctUINT32 waitSize; 1835 gctUINT32 waitSize;
1836 1836
1837 #ifdef __QNXNTO__ 1837 #ifdef __QNXNTO__
1838 gctPOINTER userCommandBufferLogical = gcvNULL; 1838 gctPOINTER userCommandBufferLogical = gcvNULL;
1839 gctBOOL userCommandBufferLogicalMapped = gcvFALSE; 1839 gctBOOL userCommandBufferLogicalMapped = gcvFALSE;
1840 #endif 1840 #endif
1841 1841
1842 #if gcdPROCESS_ADDRESS_SPACE 1842 #if gcdPROCESS_ADDRESS_SPACE
1843 gckMMU mmu; 1843 gckMMU mmu;
1844 gctUINT32 oldValue; 1844 gctUINT32 oldValue;
1845 #endif 1845 #endif
1846 1846
1847 #if gcdDUMP_COMMAND 1847 #if gcdDUMP_COMMAND
1848 gctPOINTER contextDumpLogical = gcvNULL; 1848 gctPOINTER contextDumpLogical = gcvNULL;
1849 gctSIZE_T contextDumpBytes = 0; 1849 gctSIZE_T contextDumpBytes = 0;
1850 gctPOINTER bufferDumpLogical = gcvNULL; 1850 gctPOINTER bufferDumpLogical = gcvNULL;
1851 gctSIZE_T bufferDumpBytes = 0; 1851 gctSIZE_T bufferDumpBytes = 0;
1852 # endif 1852 # endif
1853 gctUINT32 exitLinkLow = 0, exitLinkHigh = 0; 1853 gctUINT32 exitLinkLow = 0, exitLinkHigh = 0;
1854 gctUINT32 entryLinkLow = 0, entryLinkHigh = 0; 1854 gctUINT32 entryLinkLow = 0, entryLinkHigh = 0;
1855 gctUINT32 commandLinkLow = 0, commandLinkHigh = 0; 1855 gctUINT32 commandLinkLow = 0, commandLinkHigh = 0;
1856 1856
1857 gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer = gcvNULL; 1857 gckVIRTUAL_COMMAND_BUFFER_PTR virtualCommandBuffer = gcvNULL;
1858 gctUINT64 asyncCommandStamp = 0; 1858 gctUINT64 asyncCommandStamp = 0;
1859 gcoCMDBUF lastCommandBuffer = gcvNULL; 1859 gcoCMDBUF lastCommandBuffer = gcvNULL;
1860 gctPOINTER pointer = gcvNULL; 1860 gctPOINTER pointer = gcvNULL;
1861 gckKERNEL kernel = Command->kernel; 1861 gckKERNEL kernel = Command->kernel;
1862 1862
1863 gctPHYS_ADDR contextPhysHandle = gcvNULL; 1863 gctPHYS_ADDR contextPhysHandle = gcvNULL;
1864 gctPHYS_ADDR physHandle = gcvNULL; 1864 gctPHYS_ADDR physHandle = gcvNULL;
1865 #endif 1865 #endif
1866 1866
1867 gcmkHEADER_ARG( 1867 gcmkHEADER_ARG(
1868 "Command=0x%x CommandBuffer=0x%x ProcessID=%d", 1868 "Command=0x%x CommandBuffer=0x%x ProcessID=%d",
1869 Command, CommandBuffer, ProcessID 1869 Command, CommandBuffer, ProcessID
1870 ); 1870 );
1871 1871
1872 /* Verify the arguments. */ 1872 /* Verify the arguments. */
1873 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 1873 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
1874 1874
1875 #if !gcdNULL_DRIVER
1876 gcmkONERROR(_ProcessUserCommandBufferList(
1877 Command,
1878 CommandBuffer,
1879 &lastCommandBuffer
1880 ));
1881 #endif
1882
1883 #if gcdPROCESS_ADDRESS_SPACE 1875 #if gcdPROCESS_ADDRESS_SPACE
1884 gcmkONERROR(gckKERNEL_GetProcessMMU(Command->kernel, &mmu)); 1876 gcmkONERROR(gckKERNEL_GetProcessMMU(Command->kernel, &mmu));
1885 1877
1886 gcmkONERROR(gckOS_AtomicExchange(Command->os, 1878 gcmkONERROR(gckOS_AtomicExchange(Command->os,
1887 mmu->pageTableDirty[Command->kernel->core], 1879 mmu->pageTableDirty[Command->kernel->core],
1888 0, 1880 0,
1889 &oldValue)); 1881 &oldValue));
1890 #else 1882 #else
1891 #endif 1883 #endif
1892 1884
1893 /* Acquire the command queue. */ 1885 /* Acquire the command queue. */
1894 gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE)); 1886 gcmkONERROR(gckCOMMAND_EnterCommit(Command, gcvFALSE));
1895 commitEntered = gcvTRUE; 1887 commitEntered = gcvTRUE;
1888
1889 #if !gcdNULL_DRIVER
1890 gcmkONERROR(_ProcessUserCommandBufferList(
1891 Command,
1892 CommandBuffer,
1893 &lastCommandBuffer
1894 ));
1895 #endif
1896 1896
1897 /* Acquire the context switching mutex. */ 1897 /* Acquire the context switching mutex. */
1898 gcmkONERROR(gckOS_AcquireMutex( 1898 gcmkONERROR(gckOS_AcquireMutex(
1899 Command->os, Command->mutexContext, gcvINFINITE 1899 Command->os, Command->mutexContext, gcvINFINITE
1900 )); 1900 ));
1901 contextAcquired = gcvTRUE; 1901 contextAcquired = gcvTRUE;
1902 1902
1903 /* Extract the gckHARDWARE and gckEVENT objects. */ 1903 /* Extract the gckHARDWARE and gckEVENT objects. */
1904 hardware = Command->kernel->hardware; 1904 hardware = Command->kernel->hardware;
1905 1905
1906 /* Check wehther we need to copy the structures or not. */ 1906 /* Check wehther we need to copy the structures or not. */
1907 gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy)); 1907 gcmkONERROR(gckOS_QueryNeedCopy(Command->os, ProcessID, &needCopy));
1908 1908
1909 #if gcdNULL_DRIVER 1909 #if gcdNULL_DRIVER
1910 /* Context switch required? */ 1910 /* Context switch required? */
1911 if ((Context != gcvNULL) && (Command->currContext != Context)) 1911 if ((Context != gcvNULL) && (Command->currContext != Context))
1912 { 1912 {
1913 /* Yes, merge in the deltas. */ 1913 /* Yes, merge in the deltas. */
1914 gckCONTEXT_Update(Context, ProcessID, StateDelta); 1914 gckCONTEXT_Update(Context, ProcessID, StateDelta);
1915 1915
1916 /* Update the current context. */ 1916 /* Update the current context. */
1917 Command->currContext = Context; 1917 Command->currContext = Context;
1918 1918
1919 contextSwitched = gcvTRUE; 1919 contextSwitched = gcvTRUE;
1920 } 1920 }
1921 #else 1921 #else
1922 if (needCopy) 1922 if (needCopy)
1923 { 1923 {
1924 commandBufferObject = &Command->_commandBufferObject; 1924 commandBufferObject = &Command->_commandBufferObject;
1925 1925
1926 gcmkONERROR(gckOS_CopyFromUserData( 1926 gcmkONERROR(gckOS_CopyFromUserData(
1927 Command->os, 1927 Command->os,
1928 commandBufferObject, 1928 commandBufferObject,
1929 CommandBuffer, 1929 CommandBuffer,
1930 gcmSIZEOF(struct _gcoCMDBUF) 1930 gcmSIZEOF(struct _gcoCMDBUF)
1931 )); 1931 ));
1932 1932
1933 gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); 1933 gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER);
1934 } 1934 }
1935 else 1935 else
1936 { 1936 {
1937 gcmkONERROR(gckOS_MapUserPointer( 1937 gcmkONERROR(gckOS_MapUserPointer(
1938 Command->os, 1938 Command->os,
1939 CommandBuffer, 1939 CommandBuffer,
1940 gcmSIZEOF(struct _gcoCMDBUF), 1940 gcmSIZEOF(struct _gcoCMDBUF),
1941 &pointer 1941 &pointer
1942 )); 1942 ));
1943 1943
1944 commandBufferObject = pointer; 1944 commandBufferObject = pointer;
1945 1945
1946 gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER); 1946 gcmkVERIFY_OBJECT(commandBufferObject, gcvOBJ_COMMANDBUFFER);
1947 commandBufferMapped = gcvTRUE; 1947 commandBufferMapped = gcvTRUE;
1948 } 1948 }
1949 1949
1950 gcmkONERROR(_HandlePatchList(Command, commandBufferObject, needCopy, &asyncCommandStamp)); 1950 gcmkONERROR(_HandlePatchList(Command, commandBufferObject, needCopy, &asyncCommandStamp));
1951 1951
1952 /* Query the size of NOP command. */ 1952 /* Query the size of NOP command. */
1953 gcmkONERROR(gckHARDWARE_Nop( 1953 gcmkONERROR(gckHARDWARE_Nop(
1954 hardware, gcvNULL, &nopBytes 1954 hardware, gcvNULL, &nopBytes
1955 )); 1955 ));
1956 1956
1957 /* Query the size of pipe select command sequence. */ 1957 /* Query the size of pipe select command sequence. */
1958 gcmkONERROR(gckHARDWARE_PipeSelect( 1958 gcmkONERROR(gckHARDWARE_PipeSelect(
1959 hardware, gcvNULL, gcvPIPE_3D, &pipeBytes 1959 hardware, gcvNULL, gcvPIPE_3D, &pipeBytes
1960 )); 1960 ));
1961 1961
1962 /* Query the size of LINK command. */ 1962 /* Query the size of LINK command. */
1963 gcmkONERROR(gckHARDWARE_Link( 1963 gcmkONERROR(gckHARDWARE_Link(
1964 hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL 1964 hardware, gcvNULL, 0, 0, &linkBytes, gcvNULL, gcvNULL
1965 )); 1965 ));
1966 1966
1967 /* Compute the command buffer entry and the size. */ 1967 /* Compute the command buffer entry and the size. */
1968 commandBufferLogical 1968 commandBufferLogical
1969 = (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical) 1969 = (gctUINT8_PTR) gcmUINT64_TO_PTR(commandBufferObject->logical)
1970 + commandBufferObject->startOffset; 1970 + commandBufferObject->startOffset;
1971 1971
1972 1972
1973 /* Get the hardware address. */ 1973 /* Get the hardware address. */
1974 if (Command->kernel->virtualCommandBuffer) 1974 if (Command->kernel->virtualCommandBuffer)
1975 { 1975 {
1976 virtualCommandBuffer = gcmNAME_TO_PTR(commandBufferObject->physical); 1976 virtualCommandBuffer = gcmNAME_TO_PTR(commandBufferObject->physical);
1977 physHandle = virtualCommandBuffer->virtualBuffer.physical; 1977 physHandle = virtualCommandBuffer->virtualBuffer.physical;
1978 1978
1979 if (virtualCommandBuffer == gcvNULL) 1979 if (virtualCommandBuffer == gcvNULL)
1980 { 1980 {
1981 gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT); 1981 gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
1982 } 1982 }
1983 1983
1984 gcmkONERROR(gckKERNEL_GetGPUAddress( 1984 gcmkONERROR(gckKERNEL_GetGPUAddress(
1985 Command->kernel, 1985 Command->kernel,
1986 commandBufferLogical, 1986 commandBufferLogical,
1987 gcvTRUE, 1987 gcvTRUE,
1988 virtualCommandBuffer, 1988 virtualCommandBuffer,
1989 &commandBufferAddress 1989 &commandBufferAddress
1990 )); 1990 ));
1991 } 1991 }
1992 else 1992 else
1993 { 1993 {
1994 physHandle = gcmNAME_TO_PTR(commandBufferObject->physical); 1994 physHandle = gcmNAME_TO_PTR(commandBufferObject->physical);
1995 1995
1996 gcmkONERROR(gckHARDWARE_ConvertLogical( 1996 gcmkONERROR(gckHARDWARE_ConvertLogical(
1997 hardware, 1997 hardware,
1998 commandBufferLogical, 1998 commandBufferLogical,
1999 gcvTRUE, 1999 gcvTRUE,
2000 &commandBufferAddress 2000 &commandBufferAddress
2001 )); 2001 ));
2002 } 2002 }
2003 2003
2004 #ifdef __QNXNTO__ 2004 #ifdef __QNXNTO__
2005 userCommandBufferLogical = (gctPOINTER) commandBufferLogical; 2005 userCommandBufferLogical = (gctPOINTER) commandBufferLogical;
2006 2006
2007 gcmkONERROR(gckOS_MapUserPointer( 2007 gcmkONERROR(gckOS_MapUserPointer(
2008 Command->os, 2008 Command->os,
2009 userCommandBufferLogical, 2009 userCommandBufferLogical,
2010 0, 2010 0,
2011 &pointer)); 2011 &pointer));
2012 2012
2013 commandBufferLogical = pointer; 2013 commandBufferLogical = pointer;
2014 2014
2015 userCommandBufferLogicalMapped = gcvTRUE; 2015 userCommandBufferLogicalMapped = gcvTRUE;
2016 2016
2017 gcmkONERROR(gckOS_GetPhysicalAddress( 2017 gcmkONERROR(gckOS_GetPhysicalAddress(
2018 Command->os, 2018 Command->os,
2019 commandBufferLogical, 2019 commandBufferLogical,
2020 &commandBufferPhysical 2020 &commandBufferPhysical
2021 )); 2021 ));
2022 2022
2023 #else 2023 #else
2024 /* Get the physical address. */ 2024 /* Get the physical address. */
2025 gcmkONERROR(gckOS_UserLogicalToPhysical( 2025 gcmkONERROR(gckOS_UserLogicalToPhysical(
2026 Command->os, 2026 Command->os,
2027 commandBufferLogical, 2027 commandBufferLogical,
2028 &commandBufferPhysical 2028 &commandBufferPhysical
2029 )); 2029 ));
2030 #endif 2030 #endif
2031 2031
2032 commandBufferSize 2032 commandBufferSize
2033 = commandBufferObject->offset 2033 = commandBufferObject->offset
2034 + commandBufferObject->reservedTail 2034 + commandBufferObject->reservedTail
2035 - commandBufferObject->startOffset; 2035 - commandBufferObject->startOffset;
2036 2036
2037 gcmkONERROR(_FlushMMU(Command)); 2037 gcmkONERROR(_FlushMMU(Command));
2038 2038
2039 if (Command->dummyDraw == gcvTRUE && 2039 if (Command->dummyDraw == gcvTRUE &&
2040 Context != gcvNULL) 2040 Context != gcvNULL)
2041 { 2041 {
2042 Command->dummyDraw = gcvFALSE; 2042 Command->dummyDraw = gcvFALSE;
2043 gcmkONERROR(_DummyDraw(Command)); 2043 gcmkONERROR(_DummyDraw(Command));
2044 } 2044 }
2045 2045
2046 if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && asyncCommandStamp != 0) 2046 if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT) && asyncCommandStamp != 0)
2047 { 2047 {
2048 gcmkONERROR(_WaitForAsyncCommandStamp(Command, asyncCommandStamp)); 2048 gcmkONERROR(_WaitForAsyncCommandStamp(Command, asyncCommandStamp));
2049 } 2049 }
2050 2050
2051 /* Get the current offset. */ 2051 /* Get the current offset. */
2052 offset = Command->offset; 2052 offset = Command->offset;
2053 2053
2054 /* Compute number of bytes left in current kernel command queue. */ 2054 /* Compute number of bytes left in current kernel command queue. */
2055 bytes = Command->pageSize - offset; 2055 bytes = Command->pageSize - offset;
2056 2056
2057 /* Query the size of WAIT/LINK command sequence. */ 2057 /* Query the size of WAIT/LINK command sequence. */
2058 gcmkONERROR(gckHARDWARE_WaitLink( 2058 gcmkONERROR(gckHARDWARE_WaitLink(
2059 hardware, 2059 hardware,
2060 gcvNULL, 2060 gcvNULL,
2061 ~0U, 2061 ~0U,
2062 offset, 2062 offset,
2063 &waitLinkBytes, 2063 &waitLinkBytes,
2064 gcvNULL, 2064 gcvNULL,
2065 gcvNULL 2065 gcvNULL
2066 )); 2066 ));
2067 2067
2068 /* Is there enough space in the current command queue? */ 2068 /* Is there enough space in the current command queue? */
2069 if (bytes < waitLinkBytes) 2069 if (bytes < waitLinkBytes)
2070 { 2070 {
2071 /* No, create a new one. */ 2071 /* No, create a new one. */
2072 gcmkONERROR(_NewQueue(Command, gcvFALSE)); 2072 gcmkONERROR(_NewQueue(Command, gcvFALSE));
2073 2073
2074 /* Get the new current offset. */ 2074 /* Get the new current offset. */
2075 offset = Command->offset; 2075 offset = Command->offset;
2076 2076
2077 /* Recompute the number of bytes in the new kernel command queue. */ 2077 /* Recompute the number of bytes in the new kernel command queue. */
2078 bytes = Command->pageSize - offset; 2078 bytes = Command->pageSize - offset;
2079 gcmkASSERT(bytes >= waitLinkBytes); 2079 gcmkASSERT(bytes >= waitLinkBytes);
2080 } 2080 }
2081 2081
2082 /* Compute the location if WAIT/LINK command sequence. */ 2082 /* Compute the location if WAIT/LINK command sequence. */
2083 waitLinkPhysical = Command->physical + offset; 2083 waitLinkPhysical = Command->physical + offset;
2084 waitLinkLogical = (gctUINT8_PTR) Command->logical + offset; 2084 waitLinkLogical = (gctUINT8_PTR) Command->logical + offset;
2085 waitLinkAddress = Command->address + offset; 2085 waitLinkAddress = Command->address + offset;
2086 2086
2087 /* Context switch required? */ 2087 /* Context switch required? */
2088 if (Context == gcvNULL) 2088 if (Context == gcvNULL)
2089 { 2089 {
2090 /* See if we have to switch pipes for the command buffer. */ 2090 /* See if we have to switch pipes for the command buffer. */
2091 if (commandBufferObject->entryPipe == Command->pipeSelect) 2091 if (commandBufferObject->entryPipe == Command->pipeSelect)
2092 { 2092 {
2093 /* Skip pipe switching sequence. */ 2093 /* Skip pipe switching sequence. */
2094 offset = pipeBytes; 2094 offset = pipeBytes;
2095 } 2095 }
2096 else 2096 else
2097 { 2097 {
2098 /* The current hardware and the entry command buffer pipes 2098 /* The current hardware and the entry command buffer pipes
2099 ** are different, switch to the correct pipe. */ 2099 ** are different, switch to the correct pipe. */
2100 gcmkONERROR(gckHARDWARE_PipeSelect( 2100 gcmkONERROR(gckHARDWARE_PipeSelect(
2101 Command->kernel->hardware, 2101 Command->kernel->hardware,
2102 commandBufferLogical, 2102 commandBufferLogical,
2103 commandBufferObject->entryPipe, 2103 commandBufferObject->entryPipe,
2104 &pipeBytes 2104 &pipeBytes
2105 )); 2105 ));
2106 2106
2107 /* Do not skip pipe switching sequence. */ 2107 /* Do not skip pipe switching sequence. */
2108 offset = 0; 2108 offset = 0;
2109 } 2109 }
2110 2110
2111 /* Compute the entry. */ 2111 /* Compute the entry. */
2112 entryLogical = commandBufferLogical + offset; 2112 entryLogical = commandBufferLogical + offset;
2113 entryAddress = commandBufferAddress + offset; 2113 entryAddress = commandBufferAddress + offset;
2114 entryBytes = commandBufferSize - offset; 2114 entryBytes = commandBufferSize - offset;
2115 2115
2116 Command->currContext = gcvNULL; 2116 Command->currContext = gcvNULL;
2117 } 2117 }
2118 #if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE 2118 #if gcdDEBUG_OPTION && gcdDEBUG_FORCE_CONTEXT_UPDATE
2119 else if (1) 2119 else if (1)
2120 #else 2120 #else
2121 else if (Command->currContext != Context) 2121 else if (Command->currContext != Context)
2122 #endif 2122 #endif
2123 { 2123 {
2124 /* Get the current context buffer. */ 2124 /* Get the current context buffer. */
2125 contextBuffer = Context->buffer; 2125 contextBuffer = Context->buffer;
2126 2126
2127 /* Yes, merge in the deltas. */ 2127 /* Yes, merge in the deltas. */
2128 gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta)); 2128 gcmkONERROR(gckCONTEXT_Update(Context, ProcessID, StateDelta));
2129 2129
2130 contextSwitched = gcvTRUE; 2130 contextSwitched = gcvTRUE;
2131 2131
2132 /*************************************************************** 2132 /***************************************************************
2133 ** SWITCHING CONTEXT. 2133 ** SWITCHING CONTEXT.
2134 */ 2134 */
2135 2135
2136 /* Determine context buffer entry offset. */ 2136 /* Determine context buffer entry offset. */
2137 offset = (Command->pipeSelect == gcvPIPE_3D) 2137 offset = (Command->pipeSelect == gcvPIPE_3D)
2138 2138
2139 /* Skip pipe switching sequence. */ 2139 /* Skip pipe switching sequence. */
2140 ? Context->entryOffset3D + Context->pipeSelectBytes 2140 ? Context->entryOffset3D + Context->pipeSelectBytes
2141 2141
2142 /* Do not skip pipe switching sequence. */ 2142 /* Do not skip pipe switching sequence. */
2143 : Context->entryOffset3D; 2143 : Context->entryOffset3D;
2144 2144
2145 /* Compute the entry. */ 2145 /* Compute the entry. */
2146 entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset; 2146 entryLogical = (gctUINT8_PTR) contextBuffer->logical + offset;
2147 entryAddress = contextBuffer->address + offset; 2147 entryAddress = contextBuffer->address + offset;
2148 entryBytes = Context->bufferSize - offset; 2148 entryBytes = Context->bufferSize - offset;
2149 2149
2150 /* See if we have to switch pipes between the context 2150 /* See if we have to switch pipes between the context
2151 and command buffers. */ 2151 and command buffers. */
2152 if (commandBufferObject->entryPipe == gcvPIPE_3D) 2152 if (commandBufferObject->entryPipe == gcvPIPE_3D)
2153 { 2153 {
2154 /* Skip pipe switching sequence. */ 2154 /* Skip pipe switching sequence. */
2155 offset = pipeBytes; 2155 offset = pipeBytes;
2156 } 2156 }
2157 else 2157 else
2158 { 2158 {
2159 /* The current hardware and the initial context pipes are 2159 /* The current hardware and the initial context pipes are
2160 different, switch to the correct pipe. */ 2160 different, switch to the correct pipe. */
2161 gcmkONERROR(gckHARDWARE_PipeSelect( 2161 gcmkONERROR(gckHARDWARE_PipeSelect(
2162 Command->kernel->hardware, 2162 Command->kernel->hardware,
2163 commandBufferLogical, 2163 commandBufferLogical,
2164 commandBufferObject->entryPipe, 2164 commandBufferObject->entryPipe,
2165 &pipeBytes 2165 &pipeBytes
2166 )); 2166 ));
2167 2167
2168 /* Do not skip pipe switching sequence. */ 2168 /* Do not skip pipe switching sequence. */
2169 offset = 0; 2169 offset = 0;
2170 } 2170 }
2171 2171
2172 /* Generate a LINK from the context buffer to 2172 /* Generate a LINK from the context buffer to
2173 the command buffer. */ 2173 the command buffer. */
2174 gcmkONERROR(gckHARDWARE_Link( 2174 gcmkONERROR(gckHARDWARE_Link(
2175 hardware, 2175 hardware,
2176 contextBuffer->link3D, 2176 contextBuffer->link3D,
2177 commandBufferAddress + offset, 2177 commandBufferAddress + offset,
2178 commandBufferSize - offset, 2178 commandBufferSize - offset,
2179 &linkBytes, 2179 &linkBytes,
2180 &commandLinkLow, 2180 &commandLinkLow,
2181 &commandLinkHigh 2181 &commandLinkHigh
2182 )); 2182 ));
2183 2183
2184 #if USE_KERNEL_VIRTUAL_BUFFERS 2184 #if USE_KERNEL_VIRTUAL_BUFFERS
2185 if (Command->kernel->virtualCommandBuffer) 2185 if (Command->kernel->virtualCommandBuffer)
2186 { 2186 {
2187 gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) contextBuffer->physical; 2187 gckVIRTUAL_COMMAND_BUFFER_PTR commandBuffer = (gckVIRTUAL_COMMAND_BUFFER_PTR) contextBuffer->physical;
2188 2188
2189 contextPhysHandle = commandBuffer->virtualBuffer.physical; 2189 contextPhysHandle = commandBuffer->virtualBuffer.physical;
2190 } 2190 }
2191 else 2191 else
2192 #endif 2192 #endif
2193 { 2193 {
2194 contextPhysHandle = contextBuffer->physical; 2194 contextPhysHandle = contextBuffer->physical;
2195 } 2195 }
2196 2196
2197 /* Flush the context buffer cache. */ 2197 /* Flush the context buffer cache. */
2198 gcmkONERROR(gckOS_CacheClean( 2198 gcmkONERROR(gckOS_CacheClean(
2199 Command->os, 2199 Command->os,
2200 0, 2200 0,
2201 contextPhysHandle, 2201 contextPhysHandle,
2202 entryAddress - contextBuffer->address, 2202 entryAddress - contextBuffer->address,
2203 entryLogical, 2203 entryLogical,
2204 entryBytes 2204 entryBytes
2205 )); 2205 ));
2206 2206
2207 /* Update the current context. */ 2207 /* Update the current context. */
2208 Command->currContext = Context; 2208 Command->currContext = Context;
2209 2209
2210 #if gcdDUMP_COMMAND 2210 #if gcdDUMP_COMMAND
2211 contextDumpLogical = entryLogical; 2211 contextDumpLogical = entryLogical;
2212 contextDumpBytes = entryBytes; 2212 contextDumpBytes = entryBytes;
2213 #endif 2213 #endif
2214 2214
2215 #if gcdSECURITY 2215 #if gcdSECURITY
2216 /* Commit context buffer to trust zone. */ 2216 /* Commit context buffer to trust zone. */
2217 gckKERNEL_SecurityExecute( 2217 gckKERNEL_SecurityExecute(
2218 Command->kernel, 2218 Command->kernel,
2219 entryLogical, 2219 entryLogical,
2220 entryBytes - 8 2220 entryBytes - 8
2221 ); 2221 );
2222 #endif 2222 #endif
2223 2223
2224 #if gcdRECORD_COMMAND 2224 #if gcdRECORD_COMMAND
2225 gckRECORDER_Record( 2225 gckRECORDER_Record(
2226 Command->recorder, 2226 Command->recorder,
2227 gcvNULL, 2227 gcvNULL,
2228 0xFFFFFFFF, 2228 0xFFFFFFFF,
2229 entryLogical, 2229 entryLogical,
2230 entryBytes 2230 entryBytes
2231 ); 2231 );
2232 #endif 2232 #endif
2233 } 2233 }
2234 2234
2235 /* Same context. */ 2235 /* Same context. */
2236 else 2236 else
2237 { 2237 {
2238 /* See if we have to switch pipes for the command buffer. */ 2238 /* See if we have to switch pipes for the command buffer. */
2239 if (commandBufferObject->entryPipe == Command->pipeSelect) 2239 if (commandBufferObject->entryPipe == Command->pipeSelect)
2240 { 2240 {
2241 /* Skip pipe switching sequence. */ 2241 /* Skip pipe switching sequence. */
2242 offset = pipeBytes; 2242 offset = pipeBytes;
2243 } 2243 }
2244 else 2244 else
2245 { 2245 {
2246 /* The current hardware and the entry command buffer pipes 2246 /* The current hardware and the entry command buffer pipes
2247 ** are different, switch to the correct pipe. */ 2247 ** are different, switch to the correct pipe. */
2248 gcmkONERROR(gckHARDWARE_PipeSelect( 2248 gcmkONERROR(gckHARDWARE_PipeSelect(
2249 Command->kernel->hardware, 2249 Command->kernel->hardware,
2250 commandBufferLogical, 2250 commandBufferLogical,
2251 commandBufferObject->entryPipe, 2251 commandBufferObject->entryPipe,
2252 &pipeBytes 2252 &pipeBytes
2253 )); 2253 ));
2254 2254
2255 /* Do not skip pipe switching sequence. */ 2255 /* Do not skip pipe switching sequence. */
2256 offset = 0; 2256 offset = 0;
2257 } 2257 }
2258 2258
2259 /* Compute the entry. */ 2259 /* Compute the entry. */
2260 entryLogical = commandBufferLogical + offset; 2260 entryLogical = commandBufferLogical + offset;
2261 entryAddress = commandBufferAddress + offset; 2261 entryAddress = commandBufferAddress + offset;
2262 entryBytes = commandBufferSize - offset; 2262 entryBytes = commandBufferSize - offset;
2263 } 2263 }
2264 2264
2265 #if gcdDUMP_COMMAND 2265 #if gcdDUMP_COMMAND
2266 bufferDumpLogical = commandBufferLogical + offset; 2266 bufferDumpLogical = commandBufferLogical + offset;
2267 bufferDumpBytes = commandBufferSize - offset; 2267 bufferDumpBytes = commandBufferSize - offset;
2268 #endif 2268 #endif
2269 2269
2270 #if gcdSECURE_USER 2270 #if gcdSECURE_USER
2271 /* Process user hints. */ 2271 /* Process user hints. */
2272 gcmkONERROR(_ProcessHints(Command, ProcessID, commandBufferObject)); 2272 gcmkONERROR(_ProcessHints(Command, ProcessID, commandBufferObject));
2273 #endif 2273 #endif
2274 2274
2275 /* Determine the location to jump to for the command buffer being 2275 /* Determine the location to jump to for the command buffer being
2276 ** scheduled. */ 2276 ** scheduled. */
2277 if (Command->newQueue) 2277 if (Command->newQueue)
2278 { 2278 {
2279 /* New command queue, jump to the beginning of it. */ 2279 /* New command queue, jump to the beginning of it. */
2280 exitOffset = 0; 2280 exitOffset = 0;
2281 exitLogical = Command->logical; 2281 exitLogical = Command->logical;
2282 exitAddress = Command->address; 2282 exitAddress = Command->address;
2283 exitBytes = Command->offset + waitLinkBytes; 2283 exitBytes = Command->offset + waitLinkBytes;
2284 } 2284 }
2285 else 2285 else
2286 { 2286 {
2287 /* Still within the preexisting command queue, jump to the new 2287 /* Still within the preexisting command queue, jump to the new
2288 WAIT/LINK command sequence. */ 2288 WAIT/LINK command sequence. */
2289 exitOffset = offset; 2289 exitOffset = offset;
2290 exitLogical = waitLinkLogical; 2290 exitLogical = waitLinkLogical;
2291 exitAddress = waitLinkAddress; 2291 exitAddress = waitLinkAddress;
2292 exitBytes = waitLinkBytes; 2292 exitBytes = waitLinkBytes;
2293 } 2293 }
2294 2294
2295 /* Add a new WAIT/LINK command sequence. When the command buffer which is 2295 /* Add a new WAIT/LINK command sequence. When the command buffer which is
2296 currently being scheduled is fully executed by the GPU, the FE will 2296 currently being scheduled is fully executed by the GPU, the FE will
2297 jump to this WAIT/LINK sequence. */ 2297 jump to this WAIT/LINK sequence. */
2298 gcmkONERROR(gckHARDWARE_WaitLink( 2298 gcmkONERROR(gckHARDWARE_WaitLink(
2299 hardware, 2299 hardware,
2300 waitLinkLogical, 2300 waitLinkLogical,
2301 waitLinkAddress, 2301 waitLinkAddress,
2302 offset, 2302 offset,
2303 &waitLinkBytes, 2303 &waitLinkBytes,
2304 &waitOffset, 2304 &waitOffset,
2305 &waitSize 2305 &waitSize
2306 )); 2306 ));
2307 2307
2308 /* Flush the command queue cache. */ 2308 /* Flush the command queue cache. */
2309 gcmkONERROR(gckOS_CacheClean( 2309 gcmkONERROR(gckOS_CacheClean(
2310 Command->os, 2310 Command->os,
2311 0, 2311 0,
2312 Command->physHandle, 2312 Command->physHandle,
2313 exitOffset, 2313 exitOffset,
2314 exitLogical, 2314 exitLogical,
2315 exitBytes 2315 exitBytes
2316 )); 2316 ));
2317 2317
2318 /* Determine the location of the TAIL in the command buffer. */ 2318 /* Determine the location of the TAIL in the command buffer. */
2319 commandBufferTail 2319 commandBufferTail
2320 = commandBufferLogical 2320 = commandBufferLogical
2321 + commandBufferSize 2321 + commandBufferSize
2322 - commandBufferObject->reservedTail; 2322 - commandBufferObject->reservedTail;
2323 2323
2324 /* Generate command which writes out commit stamp. */ 2324 /* Generate command which writes out commit stamp. */
2325 if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT)) 2325 if (gckHARDWARE_IsFeatureAvailable(hardware, gcvFEATURE_FENCE_64BIT))
2326 { 2326 {
2327 gctUINT32 bytes; 2327 gctUINT32 bytes;
2328 2328
2329 gcmkONERROR(gckHARDWARE_Fence( 2329 gcmkONERROR(gckHARDWARE_Fence(
2330 hardware, 2330 hardware,
2331 gcvENGINE_RENDER, 2331 gcvENGINE_RENDER,
2332 commandBufferTail, 2332 commandBufferTail,
2333 Command->fence->address, 2333 Command->fence->address,
2334 Command->commitStamp, 2334 Command->commitStamp,
2335 &bytes 2335 &bytes
2336 )); 2336 ));
2337 2337
2338 commandBufferTail += gcdRENDER_FENCE_LENGTH; 2338 commandBufferTail += gcdRENDER_FENCE_LENGTH;
2339 } 2339 }
2340 2340
2341 /* Generate a LINK from the end of the command buffer being scheduled 2341 /* Generate a LINK from the end of the command buffer being scheduled
2342 back to the kernel command queue. */ 2342 back to the kernel command queue. */
2343 #if !gcdSECURITY 2343 #if !gcdSECURITY
2344 if (Shared == gcvFALSE) 2344 if (Shared == gcvFALSE)
2345 { 2345 {
2346 gcmkONERROR(gckHARDWARE_Link( 2346 gcmkONERROR(gckHARDWARE_Link(
2347 hardware, 2347 hardware,
2348 commandBufferTail, 2348 commandBufferTail,
2349 exitAddress, 2349 exitAddress,
2350 exitBytes, 2350 exitBytes,
2351 &linkBytes, 2351 &linkBytes,
2352 &exitLinkLow, 2352 &exitLinkLow,
2353 &exitLinkHigh 2353 &exitLinkHigh
2354 )); 2354 ));
2355 } 2355 }
2356 else 2356 else
2357 { 2357 {
2358 gctUINT8_PTR link = commandBufferTail + Index * 16; 2358 gctUINT8_PTR link = commandBufferTail + Index * 16;
2359 gctSIZE_T bytes = 8; 2359 gctSIZE_T bytes = 8;
2360 2360
2361 gcmkONERROR(gckHARDWARE_ChipEnable( 2361 gcmkONERROR(gckHARDWARE_ChipEnable(
2362 hardware, 2362 hardware,
2363 link, 2363 link,
2364 (gceCORE_3D_MASK)(1 << hardware->kernel->chipID), 2364 (gceCORE_3D_MASK)(1 << hardware->kernel->chipID),
2365 &bytes 2365 &bytes
2366 )); 2366 ));
2367 2367
2368 link += bytes; 2368 link += bytes;
2369 2369
2370 gcmkONERROR(gckHARDWARE_Link( 2370 gcmkONERROR(gckHARDWARE_Link(
2371 hardware, 2371 hardware,
2372 link, 2372 link,
2373 exitAddress, 2373 exitAddress,
2374 exitBytes, 2374 exitBytes,
2375 &linkBytes, 2375 &linkBytes,
2376 &exitLinkLow, 2376 &exitLinkLow,
2377 &exitLinkHigh 2377 &exitLinkHigh
2378 )); 2378 ));
2379 2379
2380 link += linkBytes; 2380 link += linkBytes;
2381 } 2381 }
2382 #endif 2382 #endif
2383 2383
2384 /* Flush the command buffer cache. */ 2384 /* Flush the command buffer cache. */
2385 gcmkONERROR(gckOS_CacheClean( 2385 gcmkONERROR(gckOS_CacheClean(
2386 Command->os, 2386 Command->os,
2387 ProcessID, 2387 ProcessID,
2388 physHandle, 2388 physHandle,
2389 commandBufferObject->startOffset, 2389 commandBufferObject->startOffset,
2390 commandBufferLogical, 2390 commandBufferLogical,
2391 commandBufferSize 2391 commandBufferSize
2392 )); 2392 ));
2393 2393
2394 #if gcdRECORD_COMMAND 2394 #if gcdRECORD_COMMAND
2395 gckRECORDER_Record( 2395 gckRECORDER_Record(
2396 Command->recorder, 2396 Command->recorder,
2397 commandBufferLogical + offset, 2397 commandBufferLogical + offset,
2398 commandBufferSize - offset, 2398 commandBufferSize - offset,
2399 gcvNULL, 2399 gcvNULL,
2400 0xFFFFFFFF 2400 0xFFFFFFFF
2401 ); 2401 );
2402 2402
2403 gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp); 2403 gckRECORDER_AdvanceIndex(Command->recorder, Command->commitStamp);
2404 #endif 2404 #endif
2405 2405
2406 #if gcdSECURITY 2406 #if gcdSECURITY
2407 /* Submit command buffer to trust zone. */ 2407 /* Submit command buffer to trust zone. */
2408 gckKERNEL_SecurityExecute( 2408 gckKERNEL_SecurityExecute(
2409 Command->kernel, 2409 Command->kernel,
2410 commandBufferLogical + offset, 2410 commandBufferLogical + offset,
2411 commandBufferSize - offset - 8 2411 commandBufferSize - offset - 8
2412 ); 2412 );
2413 #else 2413 #else
2414 /* Generate a LINK from the previous WAIT/LINK command sequence to the 2414 /* Generate a LINK from the previous WAIT/LINK command sequence to the
2415 entry determined above (either the context or the command buffer). 2415 entry determined above (either the context or the command buffer).
2416 This LINK replaces the WAIT instruction from the previous WAIT/LINK 2416 This LINK replaces the WAIT instruction from the previous WAIT/LINK
2417 pair, therefore we use WAIT metrics for generation of this LINK. 2417 pair, therefore we use WAIT metrics for generation of this LINK.
2418 This action will execute the entire sequence. */ 2418 This action will execute the entire sequence. */
2419 gcmkONERROR(gckHARDWARE_Link( 2419 gcmkONERROR(gckHARDWARE_Link(
2420 hardware, 2420 hardware,
2421 Command->waitLogical, 2421 Command->waitLogical,
2422 entryAddress, 2422 entryAddress,
2423 entryBytes, 2423 entryBytes,
2424 &Command->waitSize, 2424 &Command->waitSize,
2425 &entryLinkLow, 2425 &entryLinkLow,
2426 &entryLinkHigh 2426 &entryLinkHigh
2427 )); 2427 ));
2428 #endif 2428 #endif
2429 2429
2430 #if gcdLINK_QUEUE_SIZE 2430 #if gcdLINK_QUEUE_SIZE
2431 if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND) 2431 if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_USER_COMMAND)
2432 { 2432 {
2433 gcuQUEUEDATA data; 2433 gcuQUEUEDATA data;
2434 2434
2435 gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); 2435 gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid));
2436 2436
2437 data.linkData.start = entryAddress; 2437 data.linkData.start = entryAddress;
2438 data.linkData.end = entryAddress + entryBytes; 2438 data.linkData.end = entryAddress + entryBytes;
2439 data.linkData.linkLow = entryLinkLow; 2439 data.linkData.linkLow = entryLinkLow;
2440 data.linkData.linkHigh = entryLinkHigh; 2440 data.linkData.linkHigh = entryLinkHigh;
2441 2441
2442 gckQUEUE_Enqueue(&hardware->linkQueue, &data); 2442 gckQUEUE_Enqueue(&hardware->linkQueue, &data);
2443 2443
2444 if (commandBufferAddress + offset != entryAddress) 2444 if (commandBufferAddress + offset != entryAddress)
2445 { 2445 {
2446 data.linkData.start = commandBufferAddress + offset; 2446 data.linkData.start = commandBufferAddress + offset;
2447 data.linkData.end = commandBufferAddress + commandBufferSize; 2447 data.linkData.end = commandBufferAddress + commandBufferSize;
2448 data.linkData.linkLow = commandLinkLow; 2448 data.linkData.linkLow = commandLinkLow;
2449 data.linkData.linkHigh = commandLinkHigh; 2449 data.linkData.linkHigh = commandLinkHigh;
2450 2450
2451 gckQUEUE_Enqueue(&hardware->linkQueue, &data); 2451 gckQUEUE_Enqueue(&hardware->linkQueue, &data);
2452 } 2452 }
2453 2453
2454 if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) 2454 if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND)
2455 { 2455 {
2456 data.linkData.start = exitAddress; 2456 data.linkData.start = exitAddress;
2457 data.linkData.end = exitAddress + exitBytes; 2457 data.linkData.end = exitAddress + exitBytes;
2458 data.linkData.linkLow = exitLinkLow; 2458 data.linkData.linkLow = exitLinkLow;
2459 data.linkData.linkHigh = exitLinkHigh; 2459 data.linkData.linkHigh = exitLinkHigh;
2460 2460
2461 /* Dump kernel command.*/ 2461 /* Dump kernel command.*/
2462 gckQUEUE_Enqueue(&hardware->linkQueue, &data); 2462 gckQUEUE_Enqueue(&hardware->linkQueue, &data);
2463 } 2463 }
2464 } 2464 }
2465 #endif 2465 #endif
2466 2466
2467 /* Flush the cache for the link. */ 2467 /* Flush the cache for the link. */
2468 gcmkONERROR(gckOS_CacheClean( 2468 gcmkONERROR(gckOS_CacheClean(
2469 Command->os, 2469 Command->os,
2470 0, 2470 0,
2471 Command->physHandle, 2471 Command->physHandle,
2472 Command->offset + waitOffset, 2472 Command->offset + waitOffset,
2473 Command->waitLogical, 2473 Command->waitLogical,
2474 Command->waitSize 2474 Command->waitSize
2475 )); 2475 ));
2476 2476
2477 gcmkDUMPCOMMAND( 2477 gcmkDUMPCOMMAND(
2478 Command->os, 2478 Command->os,
2479 Command->waitLogical, 2479 Command->waitLogical,
2480 Command->waitSize, 2480 Command->waitSize,
2481 gcvDUMP_BUFFER_LINK, 2481 gcvDUMP_BUFFER_LINK,
2482 gcvFALSE 2482 gcvFALSE
2483 ); 2483 );
2484 2484
2485 gcmkDUMPCOMMAND( 2485 gcmkDUMPCOMMAND(
2486 Command->os, 2486 Command->os,
2487 contextDumpLogical, 2487 contextDumpLogical,
2488 contextDumpBytes, 2488 contextDumpBytes,
2489 gcvDUMP_BUFFER_CONTEXT, 2489 gcvDUMP_BUFFER_CONTEXT,
2490 gcvFALSE 2490 gcvFALSE
2491 ); 2491 );
2492 2492
2493 gcmkDUMPCOMMAND( 2493 gcmkDUMPCOMMAND(
2494 Command->os, 2494 Command->os,
2495 bufferDumpLogical, 2495 bufferDumpLogical,
2496 bufferDumpBytes, 2496 bufferDumpBytes,
2497 gcvDUMP_BUFFER_USER, 2497 gcvDUMP_BUFFER_USER,
2498 gcvFALSE 2498 gcvFALSE
2499 ); 2499 );
2500 2500
2501 gcmkDUMPCOMMAND( 2501 gcmkDUMPCOMMAND(
2502 Command->os, 2502 Command->os,
2503 waitLinkLogical, 2503 waitLinkLogical,
2504 waitLinkBytes, 2504 waitLinkBytes,
2505 gcvDUMP_BUFFER_WAITLINK, 2505 gcvDUMP_BUFFER_WAITLINK,
2506 gcvFALSE 2506 gcvFALSE
2507 ); 2507 );
2508 2508
2509 /* Update the current pipe. */ 2509 /* Update the current pipe. */
2510 Command->pipeSelect = commandBufferObject->exitPipe; 2510 Command->pipeSelect = commandBufferObject->exitPipe;
2511 2511
2512 /* Update command queue offset. */ 2512 /* Update command queue offset. */
2513 Command->offset += waitLinkBytes; 2513 Command->offset += waitLinkBytes;
2514 Command->newQueue = gcvFALSE; 2514 Command->newQueue = gcvFALSE;
2515 2515
2516 /* Update address of last WAIT. */ 2516 /* Update address of last WAIT. */
2517 Command->waitPhysical = waitLinkPhysical + waitOffset; 2517 Command->waitPhysical = waitLinkPhysical + waitOffset;
2518 Command->waitLogical = (gctUINT8_PTR)waitLinkLogical + waitOffset; 2518 Command->waitLogical = (gctUINT8_PTR)waitLinkLogical + waitOffset;
2519 Command->waitAddress = waitLinkAddress + waitOffset; 2519 Command->waitAddress = waitLinkAddress + waitOffset;
2520 Command->waitSize = waitSize; 2520 Command->waitSize = waitSize;
2521 2521
2522 /* Update queue tail pointer. */ 2522 /* Update queue tail pointer. */
2523 gcmkONERROR(gckHARDWARE_UpdateQueueTail( 2523 gcmkONERROR(gckHARDWARE_UpdateQueueTail(
2524 hardware, Command->logical, Command->offset 2524 hardware, Command->logical, Command->offset
2525 )); 2525 ));
2526 2526
2527 #if gcdDUMP_COMMAND 2527 #if gcdDUMP_COMMAND
2528 gcmkPRINT("@[kernel.commit]"); 2528 gcmkPRINT("@[kernel.commit]");
2529 #endif 2529 #endif
2530 #endif /* gcdNULL_DRIVER */ 2530 #endif /* gcdNULL_DRIVER */
2531 2531
2532 /* Release the context switching mutex. */ 2532 /* Release the context switching mutex. */
2533 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); 2533 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
2534 contextAcquired = gcvFALSE; 2534 contextAcquired = gcvFALSE;
2535 2535
2536 *CommitStamp = Command->commitStamp; 2536 *CommitStamp = Command->commitStamp;
2537 *ContextSwitched = contextSwitched; 2537 *ContextSwitched = contextSwitched;
2538 2538
2539 Command->commitStamp++; 2539 Command->commitStamp++;
2540 2540
2541 stall = gcvFALSE; 2541 stall = gcvFALSE;
2542 2542
2543 #if gcdLINK_QUEUE_SIZE 2543 #if gcdLINK_QUEUE_SIZE
2544 if (Command->kernel->stuckDump == gcvSTUCK_DUMP_STALL_COMMAND) 2544 if (Command->kernel->stuckDump == gcvSTUCK_DUMP_STALL_COMMAND)
2545 { 2545 {
2546 if ((Command->commitStamp % (gcdLINK_QUEUE_SIZE/2)) == 0) 2546 if ((Command->commitStamp % (gcdLINK_QUEUE_SIZE/2)) == 0)
2547 { 2547 {
2548 /* If only context buffer and command buffer is recorded, 2548 /* If only context buffer and command buffer is recorded,
2549 ** each commit costs 2 slot in queue, to make sure command 2549 ** each commit costs 2 slot in queue, to make sure command
2550 ** causing stuck is recorded, number of pending command buffer 2550 ** causing stuck is recorded, number of pending command buffer
2551 ** is limited to (gckLINK_QUEUE_SIZE/2) 2551 ** is limited to (gckLINK_QUEUE_SIZE/2)
2552 */ 2552 */
2553 stall = gcvTRUE; 2553 stall = gcvTRUE;
2554 } 2554 }
2555 } 2555 }
2556 #endif 2556 #endif
2557 2557
2558 /* Release the command queue. */ 2558 /* Release the command queue. */
2559 gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE)); 2559 gcmkONERROR(gckCOMMAND_ExitCommit(Command, gcvFALSE));
2560 commitEntered = gcvFALSE; 2560 commitEntered = gcvFALSE;
2561 2561
2562 if (status == gcvSTATUS_INTERRUPTED) 2562 if (status == gcvSTATUS_INTERRUPTED)
2563 { 2563 {
2564 gcmkTRACE( 2564 gcmkTRACE(
2565 gcvLEVEL_INFO, 2565 gcvLEVEL_INFO,
2566 "%s(%d): Intterupted in gckEVENT_Submit", 2566 "%s(%d): Intterupted in gckEVENT_Submit",
2567 __FUNCTION__, __LINE__ 2567 __FUNCTION__, __LINE__
2568 ); 2568 );
2569 status = gcvSTATUS_OK; 2569 status = gcvSTATUS_OK;
2570 } 2570 }
2571 else 2571 else
2572 { 2572 {
2573 gcmkONERROR(status); 2573 gcmkONERROR(status);
2574 } 2574 }
2575 2575
2576 #ifdef __QNXNTO__ 2576 #ifdef __QNXNTO__
2577 if (userCommandBufferLogicalMapped) 2577 if (userCommandBufferLogicalMapped)
2578 { 2578 {
2579 gcmkONERROR(gckOS_UnmapUserPointer( 2579 gcmkONERROR(gckOS_UnmapUserPointer(
2580 Command->os, 2580 Command->os,
2581 userCommandBufferLogical, 2581 userCommandBufferLogical,
2582 0, 2582 0,
2583 commandBufferLogical)); 2583 commandBufferLogical));
2584 2584
2585 userCommandBufferLogicalMapped = gcvFALSE; 2585 userCommandBufferLogicalMapped = gcvFALSE;
2586 } 2586 }
2587 #endif 2587 #endif
2588 2588
2589 /* Unmap the command buffer pointer. */ 2589 /* Unmap the command buffer pointer. */
2590 if (commandBufferMapped) 2590 if (commandBufferMapped)
2591 { 2591 {
2592 gcmkONERROR(gckOS_UnmapUserPointer( 2592 gcmkONERROR(gckOS_UnmapUserPointer(
2593 Command->os, 2593 Command->os,
2594 CommandBuffer, 2594 CommandBuffer,
2595 gcmSIZEOF(struct _gcoCMDBUF), 2595 gcmSIZEOF(struct _gcoCMDBUF),
2596 commandBufferObject 2596 commandBufferObject
2597 )); 2597 ));
2598 2598
2599 commandBufferMapped = gcvFALSE; 2599 commandBufferMapped = gcvFALSE;
2600 } 2600 }
2601 2601
2602 /* Return status. */ 2602 /* Return status. */
2603 gcmkFOOTER(); 2603 gcmkFOOTER();
2604 return gcvSTATUS_OK; 2604 return gcvSTATUS_OK;
2605 2605
2606 OnError: 2606 OnError:
2607 if (contextAcquired) 2607 if (contextAcquired)
2608 { 2608 {
2609 /* Release the context switching mutex. */ 2609 /* Release the context switching mutex. */
2610 gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); 2610 gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
2611 } 2611 }
2612 2612
2613 if (commitEntered) 2613 if (commitEntered)
2614 { 2614 {
2615 /* Release the command queue mutex. */ 2615 /* Release the command queue mutex. */
2616 gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE)); 2616 gcmkVERIFY_OK(gckCOMMAND_ExitCommit(Command, gcvFALSE));
2617 } 2617 }
2618 2618
2619 #ifdef __QNXNTO__ 2619 #ifdef __QNXNTO__
2620 if (userCommandBufferLogicalMapped) 2620 if (userCommandBufferLogicalMapped)
2621 { 2621 {
2622 gcmkVERIFY_OK(gckOS_UnmapUserPointer( 2622 gcmkVERIFY_OK(gckOS_UnmapUserPointer(
2623 Command->os, 2623 Command->os,
2624 userCommandBufferLogical, 2624 userCommandBufferLogical,
2625 0, 2625 0,
2626 commandBufferLogical)); 2626 commandBufferLogical));
2627 } 2627 }
2628 #endif 2628 #endif
2629 2629
2630 /* Unmap the command buffer pointer. */ 2630 /* Unmap the command buffer pointer. */
2631 if (commandBufferMapped) 2631 if (commandBufferMapped)
2632 { 2632 {
2633 gcmkVERIFY_OK(gckOS_UnmapUserPointer( 2633 gcmkVERIFY_OK(gckOS_UnmapUserPointer(
2634 Command->os, 2634 Command->os,
2635 CommandBuffer, 2635 CommandBuffer,
2636 gcmSIZEOF(struct _gcoCMDBUF), 2636 gcmSIZEOF(struct _gcoCMDBUF),
2637 commandBufferObject 2637 commandBufferObject
2638 )); 2638 ));
2639 } 2639 }
2640 2640
2641 /* Return status. */ 2641 /* Return status. */
2642 gcmkFOOTER(); 2642 gcmkFOOTER();
2643 return status; 2643 return status;
2644 } 2644 }
2645 2645
2646 /******************************************************************************* 2646 /*******************************************************************************
2647 ** 2647 **
2648 ** gckCOMMAND_Reserve 2648 ** gckCOMMAND_Reserve
2649 ** 2649 **
2650 ** Reserve space in the command queue. Also acquire the command queue mutex. 2650 ** Reserve space in the command queue. Also acquire the command queue mutex.
2651 ** 2651 **
2652 ** INPUT: 2652 ** INPUT:
2653 ** 2653 **
2654 ** gckCOMMAND Command 2654 ** gckCOMMAND Command
2655 ** Pointer to an gckCOMMAND object. 2655 ** Pointer to an gckCOMMAND object.
2656 ** 2656 **
2657 ** gctSIZE_T RequestedBytes 2657 ** gctSIZE_T RequestedBytes
2658 ** Number of bytes previously reserved. 2658 ** Number of bytes previously reserved.
2659 ** 2659 **
2660 ** OUTPUT: 2660 ** OUTPUT:
2661 ** 2661 **
2662 ** gctPOINTER * Buffer 2662 ** gctPOINTER * Buffer
2663 ** Pointer to a variable that will receive the address of the reserved 2663 ** Pointer to a variable that will receive the address of the reserved
2664 ** space. 2664 ** space.
2665 ** 2665 **
2666 ** gctSIZE_T * BufferSize 2666 ** gctSIZE_T * BufferSize
2667 ** Pointer to a variable that will receive the number of bytes 2667 ** Pointer to a variable that will receive the number of bytes
2668 ** available in the command queue. 2668 ** available in the command queue.
2669 */ 2669 */
2670 gceSTATUS 2670 gceSTATUS
2671 gckCOMMAND_Reserve( 2671 gckCOMMAND_Reserve(
2672 IN gckCOMMAND Command, 2672 IN gckCOMMAND Command,
2673 IN gctUINT32 RequestedBytes, 2673 IN gctUINT32 RequestedBytes,
2674 OUT gctPOINTER * Buffer, 2674 OUT gctPOINTER * Buffer,
2675 OUT gctUINT32 * BufferSize 2675 OUT gctUINT32 * BufferSize
2676 ) 2676 )
2677 { 2677 {
2678 gceSTATUS status; 2678 gceSTATUS status;
2679 gctUINT32 bytes; 2679 gctUINT32 bytes;
2680 gctUINT32 requiredBytes; 2680 gctUINT32 requiredBytes;
2681 gctUINT32 requestedAligned; 2681 gctUINT32 requestedAligned;
2682 2682
2683 gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); 2683 gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes);
2684 2684
2685 /* Verify the arguments. */ 2685 /* Verify the arguments. */
2686 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 2686 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
2687 2687
2688 /* Compute aligned number of reuested bytes. */ 2688 /* Compute aligned number of reuested bytes. */
2689 requestedAligned = gcmALIGN(RequestedBytes, Command->alignment); 2689 requestedAligned = gcmALIGN(RequestedBytes, Command->alignment);
2690 2690
2691 /* Another WAIT/LINK command sequence will have to be appended after 2691 /* Another WAIT/LINK command sequence will have to be appended after
2692 the requested area being reserved. Compute the number of bytes 2692 the requested area being reserved. Compute the number of bytes
2693 required for WAIT/LINK at the location after the reserved area. */ 2693 required for WAIT/LINK at the location after the reserved area. */
2694 gcmkONERROR(gckHARDWARE_WaitLink( 2694 gcmkONERROR(gckHARDWARE_WaitLink(
2695 Command->kernel->hardware, 2695 Command->kernel->hardware,
2696 gcvNULL, 2696 gcvNULL,
2697 ~0U, 2697 ~0U,
2698 Command->offset + requestedAligned, 2698 Command->offset + requestedAligned,
2699 &requiredBytes, 2699 &requiredBytes,
2700 gcvNULL, 2700 gcvNULL,
2701 gcvNULL 2701 gcvNULL
2702 )); 2702 ));
2703 2703
2704 /* Compute total number of bytes required. */ 2704 /* Compute total number of bytes required. */
2705 requiredBytes += requestedAligned; 2705 requiredBytes += requestedAligned;
2706 2706
2707 /* Compute number of bytes available in command queue. */ 2707 /* Compute number of bytes available in command queue. */
2708 bytes = Command->pageSize - Command->offset; 2708 bytes = Command->pageSize - Command->offset;
2709 2709
2710 /* Is there enough space in the current command queue? */ 2710 /* Is there enough space in the current command queue? */
2711 if (bytes < requiredBytes) 2711 if (bytes < requiredBytes)
2712 { 2712 {
2713 /* Create a new command queue. */ 2713 /* Create a new command queue. */
2714 gcmkONERROR(_NewQueue(Command, gcvFALSE)); 2714 gcmkONERROR(_NewQueue(Command, gcvFALSE));
2715 2715
2716 /* Recompute the number of bytes in the new kernel command queue. */ 2716 /* Recompute the number of bytes in the new kernel command queue. */
2717 bytes = Command->pageSize - Command->offset; 2717 bytes = Command->pageSize - Command->offset;
2718 2718
2719 /* Still not enough space? */ 2719 /* Still not enough space? */
2720 if (bytes < requiredBytes) 2720 if (bytes < requiredBytes)
2721 { 2721 {
2722 /* Rare case, not enough room in command queue. */ 2722 /* Rare case, not enough room in command queue. */
2723 gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL); 2723 gcmkONERROR(gcvSTATUS_BUFFER_TOO_SMALL);
2724 } 2724 }
2725 } 2725 }
2726 2726
2727 /* Return pointer to empty slot command queue. */ 2727 /* Return pointer to empty slot command queue. */
2728 *Buffer = (gctUINT8 *) Command->logical + Command->offset; 2728 *Buffer = (gctUINT8 *) Command->logical + Command->offset;
2729 2729
2730 /* Return number of bytes left in command queue. */ 2730 /* Return number of bytes left in command queue. */
2731 *BufferSize = bytes; 2731 *BufferSize = bytes;
2732 2732
2733 /* Success. */ 2733 /* Success. */
2734 gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize); 2734 gcmkFOOTER_ARG("*Buffer=0x%x *BufferSize=%lu", *Buffer, *BufferSize);
2735 return gcvSTATUS_OK; 2735 return gcvSTATUS_OK;
2736 2736
2737 OnError: 2737 OnError:
2738 /* Return status. */ 2738 /* Return status. */
2739 gcmkFOOTER(); 2739 gcmkFOOTER();
2740 return status; 2740 return status;
2741 } 2741 }
2742 2742
2743 /******************************************************************************* 2743 /*******************************************************************************
2744 ** 2744 **
2745 ** gckCOMMAND_Execute 2745 ** gckCOMMAND_Execute
2746 ** 2746 **
2747 ** Execute a previously reserved command queue by appending a WAIT/LINK command 2747 ** Execute a previously reserved command queue by appending a WAIT/LINK command
2748 ** sequence after it and modifying the last WAIT into a LINK command. The 2748 ** sequence after it and modifying the last WAIT into a LINK command. The
2749 ** command FIFO mutex will be released whether this function succeeds or not. 2749 ** command FIFO mutex will be released whether this function succeeds or not.
2750 ** 2750 **
2751 ** INPUT: 2751 ** INPUT:
2752 ** 2752 **
2753 ** gckCOMMAND Command 2753 ** gckCOMMAND Command
2754 ** Pointer to an gckCOMMAND object. 2754 ** Pointer to an gckCOMMAND object.
2755 ** 2755 **
2756 ** gctSIZE_T RequestedBytes 2756 ** gctSIZE_T RequestedBytes
2757 ** Number of bytes previously reserved. 2757 ** Number of bytes previously reserved.
2758 ** 2758 **
2759 ** OUTPUT: 2759 ** OUTPUT:
2760 ** 2760 **
2761 ** Nothing. 2761 ** Nothing.
2762 */ 2762 */
2763 gceSTATUS 2763 gceSTATUS
2764 gckCOMMAND_Execute( 2764 gckCOMMAND_Execute(
2765 IN gckCOMMAND Command, 2765 IN gckCOMMAND Command,
2766 IN gctUINT32 RequestedBytes 2766 IN gctUINT32 RequestedBytes
2767 ) 2767 )
2768 { 2768 {
2769 gceSTATUS status; 2769 gceSTATUS status;
2770 2770
2771 gctUINT32 waitLinkPhysical; 2771 gctUINT32 waitLinkPhysical;
2772 gctUINT8_PTR waitLinkLogical; 2772 gctUINT8_PTR waitLinkLogical;
2773 gctUINT32 waitLinkAddress; 2773 gctUINT32 waitLinkAddress;
2774 gctUINT32 waitLinkOffset; 2774 gctUINT32 waitLinkOffset;
2775 gctUINT32 waitLinkBytes; 2775 gctUINT32 waitLinkBytes;
2776 2776
2777 gctUINT32 waitPhysical; 2777 gctUINT32 waitPhysical;
2778 gctPOINTER waitLogical; 2778 gctPOINTER waitLogical;
2779 gctUINT32 waitAddress; 2779 gctUINT32 waitAddress;
2780 gctUINT32 waitOffset; 2780 gctUINT32 waitOffset;
2781 gctUINT32 waitBytes; 2781 gctUINT32 waitBytes;
2782 2782
2783 gctUINT32 linkLow, linkHigh; 2783 gctUINT32 linkLow, linkHigh;
2784 2784
2785 gctUINT32 execOffset; 2785 gctUINT32 execOffset;
2786 gctPOINTER execLogical; 2786 gctPOINTER execLogical;
2787 gctUINT32 execAddress; 2787 gctUINT32 execAddress;
2788 gctUINT32 execBytes; 2788 gctUINT32 execBytes;
2789 2789
2790 gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes); 2790 gcmkHEADER_ARG("Command=0x%x RequestedBytes=%lu", Command, RequestedBytes);
2791 2791
2792 /* Verify the arguments. */ 2792 /* Verify the arguments. */
2793 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 2793 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
2794 2794
2795 /* Compute offset for WAIT/LINK. */ 2795 /* Compute offset for WAIT/LINK. */
2796 waitLinkOffset = Command->offset + RequestedBytes; 2796 waitLinkOffset = Command->offset + RequestedBytes;
2797 2797
2798 /* Compute number of bytes left in command queue. */ 2798 /* Compute number of bytes left in command queue. */
2799 waitLinkBytes = Command->pageSize - waitLinkOffset; 2799 waitLinkBytes = Command->pageSize - waitLinkOffset;
2800 2800
2801 /* Compute the location if WAIT/LINK command sequence. */ 2801 /* Compute the location if WAIT/LINK command sequence. */
2802 waitLinkPhysical = Command->physical + waitLinkOffset; 2802 waitLinkPhysical = Command->physical + waitLinkOffset;
2803 waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset; 2803 waitLinkLogical = (gctUINT8_PTR) Command->logical + waitLinkOffset;
2804 waitLinkAddress = Command->address + waitLinkOffset; 2804 waitLinkAddress = Command->address + waitLinkOffset;
2805 2805
2806 /* Append WAIT/LINK in command queue. */ 2806 /* Append WAIT/LINK in command queue. */
2807 gcmkONERROR(gckHARDWARE_WaitLink( 2807 gcmkONERROR(gckHARDWARE_WaitLink(
2808 Command->kernel->hardware, 2808 Command->kernel->hardware,
2809 waitLinkLogical, 2809 waitLinkLogical,
2810 waitLinkAddress, 2810 waitLinkAddress,
2811 waitLinkOffset, 2811 waitLinkOffset,
2812 &waitLinkBytes, 2812 &waitLinkBytes,
2813 &waitOffset, 2813 &waitOffset,
2814 &waitBytes 2814 &waitBytes
2815 )); 2815 ));
2816 2816
2817 /* Compute the location if WAIT command. */ 2817 /* Compute the location if WAIT command. */
2818 waitPhysical = waitLinkPhysical + waitOffset; 2818 waitPhysical = waitLinkPhysical + waitOffset;
2819 waitLogical = waitLinkLogical + waitOffset; 2819 waitLogical = waitLinkLogical + waitOffset;
2820 waitAddress = waitLinkAddress + waitOffset; 2820 waitAddress = waitLinkAddress + waitOffset;
2821 2821
2822 /* Determine the location to jump to for the command buffer being 2822 /* Determine the location to jump to for the command buffer being
2823 ** scheduled. */ 2823 ** scheduled. */
2824 if (Command->newQueue) 2824 if (Command->newQueue)
2825 { 2825 {
2826 /* New command queue, jump to the beginning of it. */ 2826 /* New command queue, jump to the beginning of it. */
2827 execOffset = 0; 2827 execOffset = 0;
2828 execLogical = Command->logical; 2828 execLogical = Command->logical;
2829 execAddress = Command->address; 2829 execAddress = Command->address;
2830 execBytes = waitLinkOffset + waitLinkBytes; 2830 execBytes = waitLinkOffset + waitLinkBytes;
2831 } 2831 }
2832 else 2832 else
2833 { 2833 {
2834 /* Still within the preexisting command queue, jump directly to the 2834 /* Still within the preexisting command queue, jump directly to the
2835 reserved area. */ 2835 reserved area. */
2836 execOffset = Command->offset; 2836 execOffset = Command->offset;
2837 execLogical = (gctUINT8 *) Command->logical + Command->offset; 2837 execLogical = (gctUINT8 *) Command->logical + Command->offset;
2838 execAddress = Command->address + Command->offset; 2838 execAddress = Command->address + Command->offset;
2839 execBytes = RequestedBytes + waitLinkBytes; 2839 execBytes = RequestedBytes + waitLinkBytes;
2840 } 2840 }
2841 2841
2842 /* Flush the cache. */ 2842 /* Flush the cache. */
2843 gcmkONERROR(gckOS_CacheClean( 2843 gcmkONERROR(gckOS_CacheClean(
2844 Command->os, 2844 Command->os,
2845 0, 2845 0,
2846 Command->physHandle, 2846 Command->physHandle,
2847 execOffset, 2847 execOffset,
2848 execLogical, 2848 execLogical,
2849 execBytes 2849 execBytes
2850 )); 2850 ));
2851 2851
2852 /* Convert the last WAIT into a LINK. */ 2852 /* Convert the last WAIT into a LINK. */
2853 gcmkONERROR(gckHARDWARE_Link( 2853 gcmkONERROR(gckHARDWARE_Link(
2854 Command->kernel->hardware, 2854 Command->kernel->hardware,
2855 Command->waitLogical, 2855 Command->waitLogical,
2856 execAddress, 2856 execAddress,
2857 execBytes, 2857 execBytes,
2858 &Command->waitSize, 2858 &Command->waitSize,
2859 &linkLow, 2859 &linkLow,
2860 &linkHigh 2860 &linkHigh
2861 )); 2861 ));
2862 2862
2863 /* Flush the cache. */ 2863 /* Flush the cache. */
2864 gcmkONERROR(gckOS_CacheClean( 2864 gcmkONERROR(gckOS_CacheClean(
2865 Command->os, 2865 Command->os,
2866 0, 2866 0,
2867 Command->physHandle, 2867 Command->physHandle,
2868 waitLinkOffset + waitOffset, 2868 waitLinkOffset + waitOffset,
2869 Command->waitLogical, 2869 Command->waitLogical,
2870 Command->waitSize 2870 Command->waitSize
2871 )); 2871 ));
2872 2872
2873 #if gcdLINK_QUEUE_SIZE 2873 #if gcdLINK_QUEUE_SIZE
2874 if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND) 2874 if (Command->kernel->stuckDump >= gcvSTUCK_DUMP_ALL_COMMAND)
2875 { 2875 {
2876 gcuQUEUEDATA data; 2876 gcuQUEUEDATA data;
2877 2877
2878 gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid)); 2878 gcmkVERIFY_OK(gckOS_GetProcessID(&data.linkData.pid));
2879 2879
2880 data.linkData.start = execAddress; 2880 data.linkData.start = execAddress;
2881 data.linkData.end = execAddress + execBytes; 2881 data.linkData.end = execAddress + execBytes;
2882 data.linkData.linkLow = linkLow; 2882 data.linkData.linkLow = linkLow;
2883 data.linkData.linkHigh = linkHigh; 2883 data.linkData.linkHigh = linkHigh;
2884 2884
2885 gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data); 2885 gckQUEUE_Enqueue(&Command->kernel->hardware->linkQueue, &data);
2886 } 2886 }
2887 #endif 2887 #endif
2888 2888
2889 gcmkDUMPCOMMAND( 2889 gcmkDUMPCOMMAND(
2890 Command->os, 2890 Command->os,
2891 Command->waitLogical, 2891 Command->waitLogical,
2892 Command->waitSize, 2892 Command->waitSize,
2893 gcvDUMP_BUFFER_LINK, 2893 gcvDUMP_BUFFER_LINK,
2894 gcvFALSE 2894 gcvFALSE
2895 ); 2895 );
2896 2896
2897 gcmkDUMPCOMMAND( 2897 gcmkDUMPCOMMAND(
2898 Command->os, 2898 Command->os,
2899 execLogical, 2899 execLogical,
2900 execBytes, 2900 execBytes,
2901 gcvDUMP_BUFFER_KERNEL, 2901 gcvDUMP_BUFFER_KERNEL,
2902 gcvFALSE 2902 gcvFALSE
2903 ); 2903 );
2904 2904
2905 /* Update the pointer to the last WAIT. */ 2905 /* Update the pointer to the last WAIT. */
2906 Command->waitPhysical = waitPhysical; 2906 Command->waitPhysical = waitPhysical;
2907 Command->waitLogical = waitLogical; 2907 Command->waitLogical = waitLogical;
2908 Command->waitAddress = waitAddress; 2908 Command->waitAddress = waitAddress;
2909 Command->waitSize = waitBytes; 2909 Command->waitSize = waitBytes;
2910 2910
2911 /* Update the command queue. */ 2911 /* Update the command queue. */
2912 Command->offset += RequestedBytes + waitLinkBytes; 2912 Command->offset += RequestedBytes + waitLinkBytes;
2913 Command->newQueue = gcvFALSE; 2913 Command->newQueue = gcvFALSE;
2914 2914
2915 /* Update queue tail pointer. */ 2915 /* Update queue tail pointer. */
2916 gcmkONERROR(gckHARDWARE_UpdateQueueTail( 2916 gcmkONERROR(gckHARDWARE_UpdateQueueTail(
2917 Command->kernel->hardware, Command->logical, Command->offset 2917 Command->kernel->hardware, Command->logical, Command->offset
2918 )); 2918 ));
2919 2919
2920 #if gcdDUMP_COMMAND 2920 #if gcdDUMP_COMMAND
2921 gcmkPRINT("@[kernel.execute]"); 2921 gcmkPRINT("@[kernel.execute]");
2922 #endif 2922 #endif
2923 2923
2924 /* Success. */ 2924 /* Success. */
2925 gcmkFOOTER_NO(); 2925 gcmkFOOTER_NO();
2926 return gcvSTATUS_OK; 2926 return gcvSTATUS_OK;
2927 2927
2928 OnError: 2928 OnError:
2929 /* Return the status. */ 2929 /* Return the status. */
2930 gcmkFOOTER(); 2930 gcmkFOOTER();
2931 return status; 2931 return status;
2932 } 2932 }
2933 2933
2934 /******************************************************************************* 2934 /*******************************************************************************
2935 ** 2935 **
2936 ** gckCOMMAND_Stall 2936 ** gckCOMMAND_Stall
2937 ** 2937 **
2938 ** The calling thread will be suspended until the command queue has been 2938 ** The calling thread will be suspended until the command queue has been
2939 ** completed. 2939 ** completed.
2940 ** 2940 **
2941 ** INPUT: 2941 ** INPUT:
2942 ** 2942 **
2943 ** gckCOMMAND Command 2943 ** gckCOMMAND Command
2944 ** Pointer to an gckCOMMAND object. 2944 ** Pointer to an gckCOMMAND object.
2945 ** 2945 **
2946 ** gctBOOL FromPower 2946 ** gctBOOL FromPower
2947 ** Determines whether the call originates from inside the power 2947 ** Determines whether the call originates from inside the power
2948 ** management or not. 2948 ** management or not.
2949 ** 2949 **
2950 ** OUTPUT: 2950 ** OUTPUT:
2951 ** 2951 **
2952 ** Nothing. 2952 ** Nothing.
2953 */ 2953 */
2954 gceSTATUS 2954 gceSTATUS
2955 gckCOMMAND_Stall( 2955 gckCOMMAND_Stall(
2956 IN gckCOMMAND Command, 2956 IN gckCOMMAND Command,
2957 IN gctBOOL FromPower 2957 IN gctBOOL FromPower
2958 ) 2958 )
2959 { 2959 {
2960 #if gcdNULL_DRIVER 2960 #if gcdNULL_DRIVER
2961 /* Do nothing with infinite hardware. */ 2961 /* Do nothing with infinite hardware. */
2962 return gcvSTATUS_OK; 2962 return gcvSTATUS_OK;
2963 #else 2963 #else
2964 gckOS os; 2964 gckOS os;
2965 gckHARDWARE hardware; 2965 gckHARDWARE hardware;
2966 gckEVENT eventObject; 2966 gckEVENT eventObject;
2967 gceSTATUS status; 2967 gceSTATUS status;
2968 gctSIGNAL signal = gcvNULL; 2968 gctSIGNAL signal = gcvNULL;
2969 gctUINT timer = 0; 2969 gctUINT timer = 0;
2970 2970
2971 gcmkHEADER_ARG("Command=0x%x", Command); 2971 gcmkHEADER_ARG("Command=0x%x", Command);
2972 2972
2973 /* Verify the arguments. */ 2973 /* Verify the arguments. */
2974 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 2974 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
2975 2975
2976 /* Extract the gckOS object pointer. */ 2976 /* Extract the gckOS object pointer. */
2977 os = Command->os; 2977 os = Command->os;
2978 gcmkVERIFY_OBJECT(os, gcvOBJ_OS); 2978 gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
2979 2979
2980 /* Extract the gckHARDWARE object pointer. */ 2980 /* Extract the gckHARDWARE object pointer. */
2981 hardware = Command->kernel->hardware; 2981 hardware = Command->kernel->hardware;
2982 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); 2982 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
2983 2983
2984 /* Extract the gckEVENT object pointer. */ 2984 /* Extract the gckEVENT object pointer. */
2985 eventObject = Command->kernel->eventObj; 2985 eventObject = Command->kernel->eventObj;
2986 gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT); 2986 gcmkVERIFY_OBJECT(eventObject, gcvOBJ_EVENT);
2987 2987
2988 /* Allocate the signal. */ 2988 /* Allocate the signal. */
2989 gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal)); 2989 gcmkONERROR(gckOS_CreateSignal(os, gcvTRUE, &signal));
2990 2990
2991 /* Append the EVENT command to trigger the signal. */ 2991 /* Append the EVENT command to trigger the signal. */
2992 gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL)); 2992 gcmkONERROR(gckEVENT_Signal(eventObject, signal, gcvKERNEL_PIXEL));
2993 2993
2994 /* Submit the event queue. */ 2994 /* Submit the event queue. */
2995 gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower)); 2995 gcmkONERROR(gckEVENT_Submit(eventObject, gcvTRUE, FromPower));
2996 2996
2997 #if gcdDUMP_COMMAND 2997 #if gcdDUMP_COMMAND
2998 gcmkPRINT("@[kernel.stall]"); 2998 gcmkPRINT("@[kernel.stall]");
2999 #endif 2999 #endif
3000 3000
3001 if (status == gcvSTATUS_CHIP_NOT_READY) 3001 if (status == gcvSTATUS_CHIP_NOT_READY)
3002 { 3002 {
3003 /* Error. */ 3003 /* Error. */
3004 goto OnError; 3004 goto OnError;
3005 } 3005 }
3006 3006
3007 do 3007 do
3008 { 3008 {
3009 /* Wait for the signal. */ 3009 /* Wait for the signal. */
3010 status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER); 3010 status = gckOS_WaitSignal(os, signal, !FromPower, gcdGPU_ADVANCETIMER);
3011 3011
3012 if (status == gcvSTATUS_TIMEOUT) 3012 if (status == gcvSTATUS_TIMEOUT)
3013 { 3013 {
3014 #if gcmIS_DEBUG(gcdDEBUG_CODE) 3014 #if gcmIS_DEBUG(gcdDEBUG_CODE)
3015 gctUINT32 idle; 3015 gctUINT32 idle;
3016 3016
3017 /* Read idle register. */ 3017 /* Read idle register. */
3018 gcmkVERIFY_OK(gckHARDWARE_GetIdle( 3018 gcmkVERIFY_OK(gckHARDWARE_GetIdle(
3019 hardware, gcvFALSE, &idle 3019 hardware, gcvFALSE, &idle
3020 )); 3020 ));
3021 3021
3022 gcmkTRACE( 3022 gcmkTRACE(
3023 gcvLEVEL_ERROR, 3023 gcvLEVEL_ERROR,
3024 "%s(%d): idle=%08x", 3024 "%s(%d): idle=%08x",
3025 __FUNCTION__, __LINE__, idle 3025 __FUNCTION__, __LINE__, idle
3026 ); 3026 );
3027 3027
3028 gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL)); 3028 gcmkVERIFY_OK(gckOS_MemoryBarrier(os, gcvNULL));
3029 #endif 3029 #endif
3030 3030
3031 /* Advance timer. */ 3031 /* Advance timer. */
3032 timer += gcdGPU_ADVANCETIMER; 3032 timer += gcdGPU_ADVANCETIMER;
3033 } 3033 }
3034 else if (status == gcvSTATUS_INTERRUPTED) 3034 else if (status == gcvSTATUS_INTERRUPTED)
3035 { 3035 {
3036 gcmkONERROR(gcvSTATUS_INTERRUPTED); 3036 gcmkONERROR(gcvSTATUS_INTERRUPTED);
3037 } 3037 }
3038 3038
3039 } 3039 }
3040 while (gcmIS_ERROR(status)); 3040 while (gcmIS_ERROR(status));
3041 3041
3042 /* Bail out on timeout. */ 3042 /* Bail out on timeout. */
3043 if (gcmIS_ERROR(status)) 3043 if (gcmIS_ERROR(status))
3044 { 3044 {
3045 /* Broadcast the stuck GPU. */ 3045 /* Broadcast the stuck GPU. */
3046 gcmkONERROR(gckOS_Broadcast( 3046 gcmkONERROR(gckOS_Broadcast(
3047 os, hardware, gcvBROADCAST_GPU_STUCK 3047 os, hardware, gcvBROADCAST_GPU_STUCK
3048 )); 3048 ));
3049 } 3049 }
3050 3050
3051 /* Delete the signal. */ 3051 /* Delete the signal. */
3052 gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); 3052 gcmkVERIFY_OK(gckOS_DestroySignal(os, signal));
3053 3053
3054 /* Success. */ 3054 /* Success. */
3055 gcmkFOOTER_NO(); 3055 gcmkFOOTER_NO();
3056 return gcvSTATUS_OK; 3056 return gcvSTATUS_OK;
3057 3057
3058 OnError: 3058 OnError:
3059 if (signal != gcvNULL) 3059 if (signal != gcvNULL)
3060 { 3060 {
3061 /* Free the signal. */ 3061 /* Free the signal. */
3062 gcmkVERIFY_OK(gckOS_DestroySignal(os, signal)); 3062 gcmkVERIFY_OK(gckOS_DestroySignal(os, signal));
3063 } 3063 }
3064 3064
3065 /* Return the status. */ 3065 /* Return the status. */
3066 gcmkFOOTER(); 3066 gcmkFOOTER();
3067 return status; 3067 return status;
3068 #endif 3068 #endif
3069 } 3069 }
3070 3070
3071 /******************************************************************************* 3071 /*******************************************************************************
3072 ** 3072 **
3073 ** gckCOMMAND_Attach 3073 ** gckCOMMAND_Attach
3074 ** 3074 **
3075 ** Attach user process. 3075 ** Attach user process.
3076 ** 3076 **
3077 ** INPUT: 3077 ** INPUT:
3078 ** 3078 **
3079 ** gckCOMMAND Command 3079 ** gckCOMMAND Command
3080 ** Pointer to a gckCOMMAND object. 3080 ** Pointer to a gckCOMMAND object.
3081 ** 3081 **
3082 ** gctUINT32 ProcessID 3082 ** gctUINT32 ProcessID
3083 ** Current process ID. 3083 ** Current process ID.
3084 ** 3084 **
3085 ** OUTPUT: 3085 ** OUTPUT:
3086 ** 3086 **
3087 ** gckCONTEXT * Context 3087 ** gckCONTEXT * Context
3088 ** Pointer to a variable that will receive a pointer to a new 3088 ** Pointer to a variable that will receive a pointer to a new
3089 ** gckCONTEXT object. 3089 ** gckCONTEXT object.
3090 ** 3090 **
3091 ** gctSIZE_T * StateCount 3091 ** gctSIZE_T * StateCount
3092 ** Pointer to a variable that will receive the number of states 3092 ** Pointer to a variable that will receive the number of states
3093 ** in the context buffer. 3093 ** in the context buffer.
3094 */ 3094 */
3095 #if (gcdENABLE_3D || gcdENABLE_2D) 3095 #if (gcdENABLE_3D || gcdENABLE_2D)
3096 gceSTATUS 3096 gceSTATUS
3097 gckCOMMAND_Attach( 3097 gckCOMMAND_Attach(
3098 IN gckCOMMAND Command, 3098 IN gckCOMMAND Command,
3099 OUT gckCONTEXT * Context, 3099 OUT gckCONTEXT * Context,
3100 OUT gctSIZE_T * MaxState, 3100 OUT gctSIZE_T * MaxState,
3101 OUT gctUINT32 * NumStates, 3101 OUT gctUINT32 * NumStates,
3102 IN gctUINT32 ProcessID 3102 IN gctUINT32 ProcessID
3103 ) 3103 )
3104 { 3104 {
3105 gceSTATUS status; 3105 gceSTATUS status;
3106 gctBOOL acquired = gcvFALSE; 3106 gctBOOL acquired = gcvFALSE;
3107 3107
3108 gcmkHEADER_ARG("Command=0x%x", Command); 3108 gcmkHEADER_ARG("Command=0x%x", Command);
3109 3109
3110 /* Verify the arguments. */ 3110 /* Verify the arguments. */
3111 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 3111 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
3112 3112
3113 /* Acquire the context switching mutex. */ 3113 /* Acquire the context switching mutex. */
3114 gcmkONERROR(gckOS_AcquireMutex( 3114 gcmkONERROR(gckOS_AcquireMutex(
3115 Command->os, Command->mutexContext, gcvINFINITE 3115 Command->os, Command->mutexContext, gcvINFINITE
3116 )); 3116 ));
3117 acquired = gcvTRUE; 3117 acquired = gcvTRUE;
3118 3118
3119 /* Construct a gckCONTEXT object. */ 3119 /* Construct a gckCONTEXT object. */
3120 gcmkONERROR(gckCONTEXT_Construct( 3120 gcmkONERROR(gckCONTEXT_Construct(
3121 Command->os, 3121 Command->os,
3122 Command->kernel->hardware, 3122 Command->kernel->hardware,
3123 ProcessID, 3123 ProcessID,
3124 Context 3124 Context
3125 )); 3125 ));
3126 3126
3127 /* Return the number of states in the context. */ 3127 /* Return the number of states in the context. */
3128 * MaxState = (* Context)->maxState; 3128 * MaxState = (* Context)->maxState;
3129 * NumStates = (* Context)->numStates; 3129 * NumStates = (* Context)->numStates;
3130 3130
3131 /* Release the context switching mutex. */ 3131 /* Release the context switching mutex. */
3132 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); 3132 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
3133 acquired = gcvFALSE; 3133 acquired = gcvFALSE;
3134 3134
3135 /* Success. */ 3135 /* Success. */
3136 gcmkFOOTER_ARG("*Context=0x%x", *Context); 3136 gcmkFOOTER_ARG("*Context=0x%x", *Context);
3137 return gcvSTATUS_OK; 3137 return gcvSTATUS_OK;
3138 3138
3139 OnError: 3139 OnError:
3140 /* Release mutex. */ 3140 /* Release mutex. */
3141 if (acquired) 3141 if (acquired)
3142 { 3142 {
3143 /* Release the context switching mutex. */ 3143 /* Release the context switching mutex. */
3144 gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); 3144 gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
3145 acquired = gcvFALSE; 3145 acquired = gcvFALSE;
3146 } 3146 }
3147 3147
3148 /* Return the status. */ 3148 /* Return the status. */
3149 gcmkFOOTER(); 3149 gcmkFOOTER();
3150 return status; 3150 return status;
3151 } 3151 }
3152 #endif 3152 #endif
3153 3153
3154 /******************************************************************************* 3154 /*******************************************************************************
3155 ** 3155 **
3156 ** gckCOMMAND_Detach 3156 ** gckCOMMAND_Detach
3157 ** 3157 **
3158 ** Detach user process. 3158 ** Detach user process.
3159 ** 3159 **
3160 ** INPUT: 3160 ** INPUT:
3161 ** 3161 **
3162 ** gckCOMMAND Command 3162 ** gckCOMMAND Command
3163 ** Pointer to a gckCOMMAND object. 3163 ** Pointer to a gckCOMMAND object.
3164 ** 3164 **
3165 ** gckCONTEXT Context 3165 ** gckCONTEXT Context
3166 ** Pointer to a gckCONTEXT object to be destroyed. 3166 ** Pointer to a gckCONTEXT object to be destroyed.
3167 ** 3167 **
3168 ** OUTPUT: 3168 ** OUTPUT:
3169 ** 3169 **
3170 ** Nothing. 3170 ** Nothing.
3171 */ 3171 */
3172 gceSTATUS 3172 gceSTATUS
3173 gckCOMMAND_Detach( 3173 gckCOMMAND_Detach(
3174 IN gckCOMMAND Command, 3174 IN gckCOMMAND Command,
3175 IN gckCONTEXT Context 3175 IN gckCONTEXT Context
3176 ) 3176 )
3177 { 3177 {
3178 gceSTATUS status; 3178 gceSTATUS status;
3179 gctBOOL acquired = gcvFALSE; 3179 gctBOOL acquired = gcvFALSE;
3180 3180
3181 gcmkHEADER_ARG("Command=0x%x Context=0x%x", Command, Context); 3181 gcmkHEADER_ARG("Command=0x%x Context=0x%x", Command, Context);
3182 3182
3183 /* Verify the arguments. */ 3183 /* Verify the arguments. */
3184 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND); 3184 gcmkVERIFY_OBJECT(Command, gcvOBJ_COMMAND);
3185 3185
3186 /* Acquire the context switching mutex. */ 3186 /* Acquire the context switching mutex. */
3187 gcmkONERROR(gckOS_AcquireMutex( 3187 gcmkONERROR(gckOS_AcquireMutex(
3188 Command->os, Command->mutexContext, gcvINFINITE 3188 Command->os, Command->mutexContext, gcvINFINITE
3189 )); 3189 ));
3190 acquired = gcvTRUE; 3190 acquired = gcvTRUE;
3191 3191
3192 /* Construct a gckCONTEXT object. */ 3192 /* Construct a gckCONTEXT object. */
3193 gcmkONERROR(gckCONTEXT_Destroy(Context)); 3193 gcmkONERROR(gckCONTEXT_Destroy(Context));
3194 3194
3195 if (Command->currContext == Context) 3195 if (Command->currContext == Context)
3196 { 3196 {
3197 /* Detach from gckCOMMAND object if the destoryed context is current context. */ 3197 /* Detach from gckCOMMAND object if the destoryed context is current context. */
3198 Command->currContext = gcvNULL; 3198 Command->currContext = gcvNULL;
3199 } 3199 }
3200 3200
3201 /* Release the context switching mutex. */ 3201 /* Release the context switching mutex. */
3202 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); 3202 gcmkONERROR(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
3203 acquired = gcvFALSE; 3203 acquired = gcvFALSE;
3204 3204
3205 /* Return the status. */ 3205 /* Return the status. */
3206 gcmkFOOTER(); 3206 gcmkFOOTER();
3207 return gcvSTATUS_OK; 3207 return gcvSTATUS_OK;
3208 3208
3209 OnError: 3209 OnError:
3210 /* Release mutex. */ 3210 /* Release mutex. */
3211 if (acquired) 3211 if (acquired)
3212 { 3212 {
3213 /* Release the context switching mutex. */ 3213 /* Release the context switching mutex. */
3214 gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext)); 3214 gcmkVERIFY_OK(gckOS_ReleaseMutex(Command->os, Command->mutexContext));
3215 acquired = gcvFALSE; 3215 acquired = gcvFALSE;
3216 } 3216 }
3217 3217
3218 /* Return the status. */ 3218 /* Return the status. */
3219 gcmkFOOTER(); 3219 gcmkFOOTER();
3220 return status; 3220 return status;
3221 } 3221 }
3222 3222
3223 /******************************************************************************* 3223 /*******************************************************************************
3224 ** 3224 **
3225 ** gckCOMMAND_DumpExecutingBuffer 3225 ** gckCOMMAND_DumpExecutingBuffer
3226 ** 3226 **
3227 ** Dump the command buffer which GPU is executing. 3227 ** Dump the command buffer which GPU is executing.
3228 ** 3228 **
3229 ** INPUT: 3229 ** INPUT:
3230 ** 3230 **
3231 ** gckCOMMAND Command 3231 ** gckCOMMAND Command
3232 ** Pointer to a gckCOMMAND object. 3232 ** Pointer to a gckCOMMAND object.
3233 ** 3233 **
3234 ** OUTPUT: 3234 ** OUTPUT:
3235 ** 3235 **
3236 ** Nothing. 3236 ** Nothing.
3237 */ 3237 */
3238 gceSTATUS 3238 gceSTATUS
3239 gckCOMMAND_DumpExecutingBuffer( 3239 gckCOMMAND_DumpExecutingBuffer(
3240 IN gckCOMMAND Command 3240 IN gckCOMMAND Command
3241 ) 3241 )
3242 { 3242 {
3243 gceSTATUS status; 3243 gceSTATUS status;
3244 gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL; 3244 gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL;
3245 gctUINT32 gpuAddress; 3245 gctUINT32 gpuAddress;
3246 gctSIZE_T pageCount; 3246 gctSIZE_T pageCount;
3247 gctPOINTER entry = gcvNULL; 3247 gctPOINTER entry = gcvNULL;
3248 gckOS os = Command->os; 3248 gckOS os = Command->os;
3249 gckKERNEL kernel = Command->kernel; 3249 gckKERNEL kernel = Command->kernel;
3250 gctUINT32 i; 3250 gctUINT32 i;
3251 gctUINT32 dumpRear; 3251 gctUINT32 dumpRear;
3252 gckQUEUE queue = &kernel->hardware->linkQueue; 3252 gckQUEUE queue = &kernel->hardware->linkQueue;
3253 gctSIZE_T bytes; 3253 gctSIZE_T bytes;
3254 gckLINKDATA linkData; 3254 gckLINKDATA linkData;
3255 gcuQUEUEDATA * queueData; 3255 gcuQUEUEDATA * queueData;
3256 gctUINT32 offset; 3256 gctUINT32 offset;
3257 gctPOINTER entryDump; 3257 gctPOINTER entryDump;
3258 gctUINT32 pid; 3258 gctUINT32 pid;
3259 gctUINT8 processName[24] = {0}; 3259 gctUINT8 processName[24] = {0};
3260 gctPHYS_ADDR_T cpuPhysical; 3260 gctPHYS_ADDR_T cpuPhysical;
3261 3261
3262 gcmkPRINT("**************************\n"); 3262 gcmkPRINT("**************************\n");
3263 gcmkPRINT("**** COMMAND BUF DUMP ****\n"); 3263 gcmkPRINT("**** COMMAND BUF DUMP ****\n");
3264 gcmkPRINT("**************************\n"); 3264 gcmkPRINT("**************************\n");
3265 3265
3266 gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1); 3266 gcmkPRINT(" Submitted commit stamp = %lld", Command->commitStamp - 1);
3267 gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical); 3267 gcmkPRINT(" Executed commit stamp = %lld", *(gctUINT64_PTR)Command->fence->logical);
3268 3268
3269 gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); 3269 gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress));
3270 gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress)); 3270 gcmkVERIFY_OK(gckOS_ReadRegisterEx(os, kernel->core, 0x664, &gpuAddress));
3271 3271
3272 gcmkPRINT("DMA Address 0x%08X, memory around:", gpuAddress); 3272 gcmkPRINT("DMA Address 0x%08X, memory around:", gpuAddress);
3273 3273
3274 /* Search and dump memory around DMA address. */ 3274 /* Search and dump memory around DMA address. */
3275 if (kernel->virtualCommandBuffer) 3275 if (kernel->virtualCommandBuffer)
3276 { 3276 {
3277 status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer); 3277 status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer);
3278 } 3278 }
3279 else 3279 else
3280 { 3280 {
3281 status = gcvSTATUS_OK; 3281 status = gcvSTATUS_OK;
3282 } 3282 }
3283 3283
3284 if (gcmIS_SUCCESS(status)) 3284 if (gcmIS_SUCCESS(status))
3285 { 3285 {
3286 if (kernel->virtualCommandBuffer) 3286 if (kernel->virtualCommandBuffer)
3287 { 3287 {
3288 gcmkVERIFY_OK(gckOS_CreateKernelVirtualMapping( 3288 gcmkVERIFY_OK(gckOS_CreateKernelVirtualMapping(
3289 os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, &entry, &pageCount)); 3289 os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, &entry, &pageCount));
3290 3290
3291 offset = gpuAddress - buffer->virtualBuffer.gpuAddress; 3291 offset = gpuAddress - buffer->virtualBuffer.gpuAddress;
3292 3292
3293 entryDump = entry; 3293 entryDump = entry;
3294 3294
3295 /* Dump one pages. */ 3295 /* Dump one pages. */
3296 bytes = 4096; 3296 bytes = 4096;
3297 3297
3298 /* Align to page. */ 3298 /* Align to page. */
3299 offset &= 0xfffff000; 3299 offset &= 0xfffff000;
3300 3300
3301 /* Kernel address of page where stall point stay. */ 3301 /* Kernel address of page where stall point stay. */
3302 entryDump = (gctUINT8_PTR)entryDump + offset; 3302 entryDump = (gctUINT8_PTR)entryDump + offset;
3303 3303
3304 /* Align to page. */ 3304 /* Align to page. */
3305 gpuAddress &= 0xfffff000; 3305 gpuAddress &= 0xfffff000;
3306 } 3306 }
3307 else 3307 else
3308 { 3308 {
3309 gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical)); 3309 gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical));
3310 3310
3311 gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, 4096, &entry)); 3311 gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, 4096, &entry));
3312 3312
3313 /* Align to page start. */ 3313 /* Align to page start. */
3314 entryDump = (gctPOINTER)((gctUINTPTR_T)entry & ~0xFFF); 3314 entryDump = (gctPOINTER)((gctUINTPTR_T)entry & ~0xFFF);
3315 gpuAddress = gpuAddress & ~0xFFF; 3315 gpuAddress = gpuAddress & ~0xFFF;
3316 bytes = 4096; 3316 bytes = 4096;
3317 } 3317 }
3318 3318
3319 gcmkPRINT("User Command Buffer:\n"); 3319 gcmkPRINT("User Command Buffer:\n");
3320 _DumpBuffer(entryDump, gpuAddress, bytes); 3320 _DumpBuffer(entryDump, gpuAddress, bytes);
3321 3321
3322 if (kernel->virtualCommandBuffer) 3322 if (kernel->virtualCommandBuffer)
3323 { 3323 {
3324 gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping( 3324 gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping(
3325 os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, entry)); 3325 os, buffer->virtualBuffer.physical, buffer->virtualBuffer.bytes, entry));
3326 } 3326 }
3327 else 3327 else
3328 { 3328 {
3329 gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, 4096)); 3329 gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, 4096));
3330 } 3330 }
3331 } 3331 }
3332 else 3332 else
3333 { 3333 {
3334 _DumpKernelCommandBuffer(Command); 3334 _DumpKernelCommandBuffer(Command);
3335 } 3335 }
3336 3336
3337 /* Dump link queue. */ 3337 /* Dump link queue. */
3338 if (queue->count) 3338 if (queue->count)
3339 { 3339 {
3340 gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:", 3340 gcmkPRINT("Dump Level is %d, dump %d valid record in link queue:",
3341 Command->kernel->stuckDump, queue->count); 3341 Command->kernel->stuckDump, queue->count);
3342 3342
3343 dumpRear = queue->count; 3343 dumpRear = queue->count;
3344 3344
3345 for (i = 0; i < dumpRear; i++) 3345 for (i = 0; i < dumpRear; i++)
3346 { 3346 {
3347 gckQUEUE_GetData(queue, i, &queueData); 3347 gckQUEUE_GetData(queue, i, &queueData);
3348 3348
3349 linkData = &queueData->linkData; 3349 linkData = &queueData->linkData;
3350 3350
3351 /* Get gpu address of this command buffer. */ 3351 /* Get gpu address of this command buffer. */
3352 gpuAddress = linkData->start; 3352 gpuAddress = linkData->start;
3353 bytes = linkData->end - gpuAddress; 3353 bytes = linkData->end - gpuAddress;
3354 3354
3355 pid = linkData->pid; 3355 pid = linkData->pid;
3356 3356
3357 gckOS_GetProcessNameByPid(pid, 16, processName); 3357 gckOS_GetProcessNameByPid(pid, 16, processName);
3358 3358
3359 if (kernel->virtualCommandBuffer) 3359 if (kernel->virtualCommandBuffer)
3360 { 3360 {
3361 buffer = gcvNULL; 3361 buffer = gcvNULL;
3362 3362
3363 /* Get the whole buffer. */ 3363 /* Get the whole buffer. */
3364 status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer); 3364 status = gckDEVICE_QueryGPUAddress(kernel->device, kernel, gpuAddress, &buffer);
3365 3365
3366 if (gcmIS_ERROR(status)) 3366 if (gcmIS_ERROR(status))
3367 { 3367 {
3368 /* Get kernel address of kernel command buffer. */ 3368 /* Get kernel address of kernel command buffer. */
3369 status = gckCOMMAND_AddressInKernelCommandBuffer( 3369 status = gckCOMMAND_AddressInKernelCommandBuffer(
3370 kernel->command, gpuAddress, &entry); 3370 kernel->command, gpuAddress, &entry);
3371 3371
3372 if (gcmIS_ERROR(status)) 3372 if (gcmIS_ERROR(status))
3373 { 3373 {
3374 status = gckHARDWARE_AddressInHardwareFuncions( 3374 status = gckHARDWARE_AddressInHardwareFuncions(
3375 kernel->hardware, gpuAddress, &entry); 3375 kernel->hardware, gpuAddress, &entry);
3376 3376
3377 if (gcmIS_ERROR(status)) 3377 if (gcmIS_ERROR(status))
3378 { 3378 {
3379 gcmkPRINT("Buffer [%08X - %08X] not found, may be freed", 3379 gcmkPRINT("Buffer [%08X - %08X] not found, may be freed",
3380 linkData->start, 3380 linkData->start,
3381 linkData->end); 3381 linkData->end);
3382 continue; 3382 continue;
3383 } 3383 }
3384 } 3384 }
3385 3385
3386 offset = 0; 3386 offset = 0;
3387 gcmkPRINT("Kernel Command Buffer: %08X, %08X", linkData->linkLow, linkData->linkHigh); 3387 gcmkPRINT("Kernel Command Buffer: %08X, %08X", linkData->linkLow, linkData->linkHigh);
3388 } 3388 }
3389 else 3389 else
3390 { 3390 {
3391 /* Get kernel logical for dump. */ 3391 /* Get kernel logical for dump. */
3392 if (buffer->virtualBuffer.kernelLogical) 3392 if (buffer->virtualBuffer.kernelLogical)
3393 { 3393 {
3394 /* Get kernel logical directly if it is a context buffer. */ 3394 /* Get kernel logical directly if it is a context buffer. */
3395 entry = buffer->virtualBuffer.kernelLogical; 3395 entry = buffer->virtualBuffer.kernelLogical;
3396 gcmkPRINT("Context Buffer: %08X, %08X PID:%d %s", 3396 gcmkPRINT("Context Buffer: %08X, %08X PID:%d %s",
3397 linkData->linkLow, linkData->linkHigh, linkData->pid, processName); 3397 linkData->linkLow, linkData->linkHigh, linkData->pid, processName);
3398 } 3398 }
3399 else 3399 else
3400 { 3400 {
3401 /* Make it accessiable by kernel if it is a user command buffer. */ 3401 /* Make it accessiable by kernel if it is a user command buffer. */
3402 gcmkVERIFY_OK( 3402 gcmkVERIFY_OK(
3403 gckOS_CreateKernelVirtualMapping(os, 3403 gckOS_CreateKernelVirtualMapping(os,
3404 buffer->virtualBuffer.physical, 3404 buffer->virtualBuffer.physical,
3405 buffer->virtualBuffer.bytes, 3405 buffer->virtualBuffer.bytes,
3406 &entry, 3406 &entry,
3407 &pageCount)); 3407 &pageCount));
3408 gcmkPRINT("User Command Buffer: %08X, %08X PID:%d %s", 3408 gcmkPRINT("User Command Buffer: %08X, %08X PID:%d %s",
3409 linkData->linkLow, linkData->linkHigh, linkData->pid, processName); 3409 linkData->linkLow, linkData->linkHigh, linkData->pid, processName);
3410 } 3410 }
3411 3411
3412 offset = gpuAddress - buffer->virtualBuffer.gpuAddress; 3412 offset = gpuAddress - buffer->virtualBuffer.gpuAddress;
3413 } 3413 }
3414 3414
3415 /* Dump from the entry. */ 3415 /* Dump from the entry. */
3416 _DumpBuffer((gctUINT8_PTR)entry + offset, gpuAddress, bytes); 3416 _DumpBuffer((gctUINT8_PTR)entry + offset, gpuAddress, bytes);
3417 3417
3418 /* Release kernel logical address if neccessary. */ 3418 /* Release kernel logical address if neccessary. */
3419 if (buffer && !buffer->virtualBuffer.kernelLogical) 3419 if (buffer && !buffer->virtualBuffer.kernelLogical)
3420 { 3420 {
3421 gcmkVERIFY_OK( 3421 gcmkVERIFY_OK(
3422 gckOS_DestroyKernelVirtualMapping(os, 3422 gckOS_DestroyKernelVirtualMapping(os,
3423 buffer->virtualBuffer.physical, 3423 buffer->virtualBuffer.physical,
3424 buffer->virtualBuffer.bytes, 3424 buffer->virtualBuffer.bytes,
3425 entry)); 3425 entry));
3426 } 3426 }
3427 } 3427 }
3428 else 3428 else
3429 { 3429 {
3430 gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical)); 3430 gcmkVERIFY_OK(gckOS_GPUPhysicalToCPUPhysical(os, gpuAddress, &cpuPhysical));
3431 3431
3432 gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, bytes, &entry)); 3432 gcmkVERIFY_OK(gckOS_MapPhysical(os, (gctUINT32) cpuPhysical, bytes, &entry));
3433 3433
3434 gcmkPRINT("Command Buffer: %08X, %08X PID:%d %s", 3434 gcmkPRINT("Command Buffer: %08X, %08X PID:%d %s",
3435 linkData->linkLow, linkData->linkHigh, linkData->pid, processName); 3435 linkData->linkLow, linkData->linkHigh, linkData->pid, processName);
3436 3436
3437 _DumpBuffer((gctUINT8_PTR)entry, gpuAddress, bytes); 3437 _DumpBuffer((gctUINT8_PTR)entry, gpuAddress, bytes);
3438 3438
3439 gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, bytes)); 3439 gcmkVERIFY_OK(gckOS_UnmapPhysical(os, entry, bytes));
3440 } 3440 }
3441 } 3441 }
3442 } 3442 }
3443 3443
3444 return gcvSTATUS_OK; 3444 return gcvSTATUS_OK;
3445 } 3445 }
3446 3446
3447 gceSTATUS 3447 gceSTATUS
3448 gckCOMMAND_AddressInKernelCommandBuffer( 3448 gckCOMMAND_AddressInKernelCommandBuffer(
3449 IN gckCOMMAND Command, 3449 IN gckCOMMAND Command,
3450 IN gctUINT32 Address, 3450 IN gctUINT32 Address,
3451 OUT gctPOINTER * Pointer 3451 OUT gctPOINTER * Pointer
3452 ) 3452 )
3453 { 3453 {
3454 gctINT i; 3454 gctINT i;
3455 3455
3456 for (i = 0; i < gcdCOMMAND_QUEUES; i++) 3456 for (i = 0; i < gcdCOMMAND_QUEUES; i++)
3457 { 3457 {
3458 if ((Address >= Command->queues[i].address) 3458 if ((Address >= Command->queues[i].address)
3459 && (Address < (Command->queues[i].address + Command->pageSize)) 3459 && (Address < (Command->queues[i].address + Command->pageSize))
3460 ) 3460 )
3461 { 3461 {
3462 *Pointer = (gctUINT8_PTR)Command->queues[i].logical 3462 *Pointer = (gctUINT8_PTR)Command->queues[i].logical
3463 + (Address - Command->queues[i].address) 3463 + (Address - Command->queues[i].address)
3464 ; 3464 ;
3465 3465