drivers/char/drm/radeon_state.c

   1 /* radeon_state.c -- State support for Radeon -*- linux-c -*- */
   2 /*
   3  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  21  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  22  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23  * DEALINGS IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *    Gareth Hughes <gareth@valinux.com>
  27  *    Kevin E. Martin <martin@valinux.com>
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "drm_sarea.h"
  33 #include "radeon_drm.h"
  34 #include "radeon_drv.h"
  35
  36 /* ================================================================
  37  * Helper functions for client state checking and fixup
  38  */
  39
  40 static __inline__ int radeon_check_and_fixup_offset(drm_radeon_private_t *
  41                                                     dev_priv,
  42                                                     struct drm_file * file_priv,
  43                                                     u32 *offset)
  44 {
  45         u64 off = *offset;
  46         u32 fb_end = dev_priv->fb_location + dev_priv->fb_size - 1;
  47         struct drm_radeon_driver_file_fields *radeon_priv;
  48
  49         /* Hrm ... the story of the offset ... So this function converts
  50          * the various ideas of what userland clients might have for an
  51          * offset in the card address space into an offset into the card
  52          * address space :) So with a sane client, it should just keep
  53          * the value intact and just do some boundary checking. However,
  54          * not all clients are sane. Some older clients pass us 0 based
  55          * offsets relative to the start of the framebuffer and some may
  56          * assume the AGP aperture it appended to the framebuffer, so we
  57          * try to detect those cases and fix them up.
  58          *
  59          * Note: It might be a good idea here to make sure the offset lands
  60          * in some "allowed" area to protect things like the PCIE GART...
  61          */
  62
  63         /* First, the best case, the offset already lands in either the
  64          * framebuffer or the GART mapped space
  65          */
  66         if (radeon_check_offset(dev_priv, off))
  67                 return 0;
  68
  69         /* Ok, that didn't happen... now check if we have a zero based
  70          * offset that fits in the framebuffer + gart space, apply the
  71          * magic offset we get from SETPARAM or calculated from fb_location
  72          */
  73         if (off < (dev_priv->fb_size + dev_priv->gart_size)) {
  74                 radeon_priv = file_priv->driver_priv;
  75                 off += radeon_priv->radeon_fb_delta;
  76         }
  77
  78         /* Finally, assume we aimed at a GART offset if beyond the fb */
  79         if (off > fb_end)
  80                 off = off - fb_end - 1 + dev_priv->gart_vm_start;
  81
  82         /* Now recheck and fail if out of bounds */
  83         if (radeon_check_offset(dev_priv, off)) {
  84                 DRM_DEBUG("offset fixed up to 0x%x\n", (unsigned int)off);
  85                 *offset = off;
  86                 return 0;
  87         }
  88         return -EINVAL;
  89 }
  90
  91 static __inline__ int radeon_check_and_fixup_packets(drm_radeon_private_t *
  92                                                      dev_priv,
  93                                                      struct drm_file *file_priv,
  94                                                      int id, u32 *data)
  95 {
  96         switch (id) {
  97
  98         case RADEON_EMIT_PP_MISC:
  99                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 100                     &data[(RADEON_RB3D_DEPTHOFFSET - RADEON_PP_MISC) / 4])) {
 101                         DRM_ERROR("Invalid depth buffer offset\n");
 102                         return -EINVAL;
 103                 }
 104                 break;
 105
 106         case RADEON_EMIT_PP_CNTL:
 107                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 108                     &data[(RADEON_RB3D_COLOROFFSET - RADEON_PP_CNTL) / 4])) {
 109                         DRM_ERROR("Invalid colour buffer offset\n");
 110                         return -EINVAL;
 111                 }
 112                 break;
 113
 114         case R200_EMIT_PP_TXOFFSET_0:
 115         case R200_EMIT_PP_TXOFFSET_1:
 116         case R200_EMIT_PP_TXOFFSET_2:
 117         case R200_EMIT_PP_TXOFFSET_3:
 118         case R200_EMIT_PP_TXOFFSET_4:
 119         case R200_EMIT_PP_TXOFFSET_5:
 120                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 121                                                   &data[0])) {
 122                         DRM_ERROR("Invalid R200 texture offset\n");
 123                         return -EINVAL;
 124                 }
 125                 break;
 126
 127         case RADEON_EMIT_PP_TXFILTER_0:
 128         case RADEON_EMIT_PP_TXFILTER_1:
 129         case RADEON_EMIT_PP_TXFILTER_2:
 130                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 131                     &data[(RADEON_PP_TXOFFSET_0 - RADEON_PP_TXFILTER_0) / 4])) {
 132                         DRM_ERROR("Invalid R100 texture offset\n");
 133                         return -EINVAL;
 134                 }
 135                 break;
 136
 137         case R200_EMIT_PP_CUBIC_OFFSETS_0:
 138         case R200_EMIT_PP_CUBIC_OFFSETS_1:
 139         case R200_EMIT_PP_CUBIC_OFFSETS_2:
 140         case R200_EMIT_PP_CUBIC_OFFSETS_3:
 141         case R200_EMIT_PP_CUBIC_OFFSETS_4:
 142         case R200_EMIT_PP_CUBIC_OFFSETS_5:{
 143                         int i;
 144                         for (i = 0; i < 5; i++) {
 145                                 if (radeon_check_and_fixup_offset(dev_priv,
 146                                                                   file_priv,
 147                                                                   &data[i])) {
 148                                         DRM_ERROR
 149                                             ("Invalid R200 cubic texture offset\n");
 150                                         return -EINVAL;
 151                                 }
 152                         }
 153                         break;
 154                 }
 155
 156         case RADEON_EMIT_PP_CUBIC_OFFSETS_T0:
 157         case RADEON_EMIT_PP_CUBIC_OFFSETS_T1:
 158         case RADEON_EMIT_PP_CUBIC_OFFSETS_T2:{
 159                         int i;
 160                         for (i = 0; i < 5; i++) {
 161                                 if (radeon_check_and_fixup_offset(dev_priv,
 162                                                                   file_priv,
 163                                                                   &data[i])) {
 164                                         DRM_ERROR
 165                                             ("Invalid R100 cubic texture offset\n");
 166                                         return -EINVAL;
 167                                 }
 168                         }
 169                 }
 170                 break;
 171
 172         case R200_EMIT_VAP_CTL:{
 173                         RING_LOCALS;
 174                         BEGIN_RING(2);
 175                         OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
 176                         ADVANCE_RING();
 177                 }
 178                 break;
 179
 180         case RADEON_EMIT_RB3D_COLORPITCH:
 181         case RADEON_EMIT_RE_LINE_PATTERN:
 182         case RADEON_EMIT_SE_LINE_WIDTH:
 183         case RADEON_EMIT_PP_LUM_MATRIX:
 184         case RADEON_EMIT_PP_ROT_MATRIX_0:
 185         case RADEON_EMIT_RB3D_STENCILREFMASK:
 186         case RADEON_EMIT_SE_VPORT_XSCALE:
 187         case RADEON_EMIT_SE_CNTL:
 188         case RADEON_EMIT_SE_CNTL_STATUS:
 189         case RADEON_EMIT_RE_MISC:
 190         case RADEON_EMIT_PP_BORDER_COLOR_0:
 191         case RADEON_EMIT_PP_BORDER_COLOR_1:
 192         case RADEON_EMIT_PP_BORDER_COLOR_2:
 193         case RADEON_EMIT_SE_ZBIAS_FACTOR:
 194         case RADEON_EMIT_SE_TCL_OUTPUT_VTX_FMT:
 195         case RADEON_EMIT_SE_TCL_MATERIAL_EMMISSIVE_RED:
 196         case R200_EMIT_PP_TXCBLEND_0:
 197         case R200_EMIT_PP_TXCBLEND_1:
 198         case R200_EMIT_PP_TXCBLEND_2:
 199         case R200_EMIT_PP_TXCBLEND_3:
 200         case R200_EMIT_PP_TXCBLEND_4:
 201         case R200_EMIT_PP_TXCBLEND_5:
 202         case R200_EMIT_PP_TXCBLEND_6:
 203         case R200_EMIT_PP_TXCBLEND_7:
 204         case R200_EMIT_TCL_LIGHT_MODEL_CTL_0:
 205         case R200_EMIT_TFACTOR_0:
 206         case R200_EMIT_VTX_FMT_0:
 207         case R200_EMIT_MATRIX_SELECT_0:
 208         case R200_EMIT_TEX_PROC_CTL_2:
 209         case R200_EMIT_TCL_UCP_VERT_BLEND_CTL:
 210         case R200_EMIT_PP_TXFILTER_0:
 211         case R200_EMIT_PP_TXFILTER_1:
 212         case R200_EMIT_PP_TXFILTER_2:
 213         case R200_EMIT_PP_TXFILTER_3:
 214         case R200_EMIT_PP_TXFILTER_4:
 215         case R200_EMIT_PP_TXFILTER_5:
 216         case R200_EMIT_VTE_CNTL:
 217         case R200_EMIT_OUTPUT_VTX_COMP_SEL:
 218         case R200_EMIT_PP_TAM_DEBUG3:
 219         case R200_EMIT_PP_CNTL_X:
 220         case R200_EMIT_RB3D_DEPTHXY_OFFSET:
 221         case R200_EMIT_RE_AUX_SCISSOR_CNTL:
 222         case R200_EMIT_RE_SCISSOR_TL_0:
 223         case R200_EMIT_RE_SCISSOR_TL_1:
 224         case R200_EMIT_RE_SCISSOR_TL_2:
 225         case R200_EMIT_SE_VAP_CNTL_STATUS:
 226         case R200_EMIT_SE_VTX_STATE_CNTL:
 227         case R200_EMIT_RE_POINTSIZE:
 228         case R200_EMIT_TCL_INPUT_VTX_VECTOR_ADDR_0:
 229         case R200_EMIT_PP_CUBIC_FACES_0:
 230         case R200_EMIT_PP_CUBIC_FACES_1:
 231         case R200_EMIT_PP_CUBIC_FACES_2:
 232         case R200_EMIT_PP_CUBIC_FACES_3:
 233         case R200_EMIT_PP_CUBIC_FACES_4:
 234         case R200_EMIT_PP_CUBIC_FACES_5:
 235         case RADEON_EMIT_PP_TEX_SIZE_0:
 236         case RADEON_EMIT_PP_TEX_SIZE_1:
 237         case RADEON_EMIT_PP_TEX_SIZE_2:
 238         case R200_EMIT_RB3D_BLENDCOLOR:
 239         case R200_EMIT_TCL_POINT_SPRITE_CNTL:
 240         case RADEON_EMIT_PP_CUBIC_FACES_0:
 241         case RADEON_EMIT_PP_CUBIC_FACES_1:
 242         case RADEON_EMIT_PP_CUBIC_FACES_2:
 243         case R200_EMIT_PP_TRI_PERF_CNTL:
 244         case R200_EMIT_PP_AFS_0:
 245         case R200_EMIT_PP_AFS_1:
 246         case R200_EMIT_ATF_TFACTOR:
 247         case R200_EMIT_PP_TXCTLALL_0:
 248         case R200_EMIT_PP_TXCTLALL_1:
 249         case R200_EMIT_PP_TXCTLALL_2:
 250         case R200_EMIT_PP_TXCTLALL_3:
 251         case R200_EMIT_PP_TXCTLALL_4:
 252         case R200_EMIT_PP_TXCTLALL_5:
 253         case R200_EMIT_VAP_PVS_CNTL:
 254                 /* These packets don't contain memory offsets */
 255                 break;
 256
 257         default:
 258                 DRM_ERROR("Unknown state packet ID %d\n", id);
 259                 return -EINVAL;
 260         }
 261
 262         return 0;
 263 }
 264
 265 static __inline__ int radeon_check_and_fixup_packet3(drm_radeon_private_t *
 266                                                      dev_priv,
 267                                                      struct drm_file *file_priv,
 268                                                      drm_radeon_kcmd_buffer_t *
 269                                                      cmdbuf,
 270                                                      unsigned int *cmdsz)
 271 {
 272         u32 *cmd = (u32 *) cmdbuf->buf;
 273         u32 offset, narrays;
 274         int count, i, k;
 275
 276         *cmdsz = 2 + ((cmd[0] & RADEON_CP_PACKET_COUNT_MASK) >> 16);
 277
 278         if ((cmd[0] & 0xc0000000) != RADEON_CP_PACKET3) {
 279                 DRM_ERROR("Not a type 3 packet\n");
 280                 return -EINVAL;
 281         }
 282
 283         if (4 * *cmdsz > cmdbuf->bufsz) {
 284                 DRM_ERROR("Packet size larger than size of data provided\n");
 285                 return -EINVAL;
 286         }
 287
 288         switch(cmd[0] & 0xff00) {
 289         /* XXX Are there old drivers needing other packets? */
 290
 291         case RADEON_3D_DRAW_IMMD:
 292         case RADEON_3D_DRAW_VBUF:
 293         case RADEON_3D_DRAW_INDX:
 294         case RADEON_WAIT_FOR_IDLE:
 295         case RADEON_CP_NOP:
 296         case RADEON_3D_CLEAR_ZMASK:
 297 /*      case RADEON_CP_NEXT_CHAR:
 298         case RADEON_CP_PLY_NEXTSCAN:
 299         case RADEON_CP_SET_SCISSORS: */ /* probably safe but will never need them? */
 300                 /* these packets are safe */
 301                 break;
 302
 303         case RADEON_CP_3D_DRAW_IMMD_2:
 304         case RADEON_CP_3D_DRAW_VBUF_2:
 305         case RADEON_CP_3D_DRAW_INDX_2:
 306         case RADEON_3D_CLEAR_HIZ:
 307                 /* safe but r200 only */
 308                 if (dev_priv->microcode_version != UCODE_R200) {
 309                         DRM_ERROR("Invalid 3d packet for r100-class chip\n");
 310                         return -EINVAL;
 311                 }
 312                 break;
 313
 314         case RADEON_3D_LOAD_VBPNTR:
 315                 count = (cmd[0] >> 16) & 0x3fff;
 316
 317                 if (count > 18) { /* 12 arrays max */
 318                         DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
 319                                   count);
 320                         return -EINVAL;
 321                 }
 322
 323                 /* carefully check packet contents */
 324                 narrays = cmd[1] & ~0xc000;
 325                 k = 0;
 326                 i = 2;
 327                 while ((k < narrays) && (i < (count + 2))) {
 328                         i++;            /* skip attribute field */
 329                         if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 330                                                           &cmd[i])) {
 331                                 DRM_ERROR
 332                                     ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
 333                                      k, i);
 334                                 return -EINVAL;
 335                         }
 336                         k++;
 337                         i++;
 338                         if (k == narrays)
 339                                 break;
 340                         /* have one more to process, they come in pairs */
 341                         if (radeon_check_and_fixup_offset(dev_priv,
 342                                                           file_priv, &cmd[i]))
 343                         {
 344                                 DRM_ERROR
 345                                     ("Invalid offset (k=%d i=%d) in 3D_LOAD_VBPNTR packet.\n",
 346                                      k, i);
 347                                 return -EINVAL;
 348                         }
 349                         k++;
 350                         i++;
 351                 }
 352                 /* do the counts match what we expect ? */
 353                 if ((k != narrays) || (i != (count + 2))) {
 354                         DRM_ERROR
 355                             ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
 356                               k, i, narrays, count + 1);
 357                         return -EINVAL;
 358                 }
 359                 break;
 360
 361         case RADEON_3D_RNDR_GEN_INDX_PRIM:
 362                 if (dev_priv->microcode_version != UCODE_R100) {
 363                         DRM_ERROR("Invalid 3d packet for r200-class chip\n");
 364                         return -EINVAL;
 365                 }
 366                 if (radeon_check_and_fixup_offset(dev_priv, file_priv, &cmd[1])) {
 367                                 DRM_ERROR("Invalid rndr_gen_indx offset\n");
 368                                 return -EINVAL;
 369                 }
 370                 break;
 371
 372         case RADEON_CP_INDX_BUFFER:
 373                 if (dev_priv->microcode_version != UCODE_R200) {
 374                         DRM_ERROR("Invalid 3d packet for r100-class chip\n");
 375                         return -EINVAL;
 376                 }
 377                 if ((cmd[1] & 0x8000ffff) != 0x80000810) {
 378                         DRM_ERROR("Invalid indx_buffer reg address %08X\n", cmd[1]);
 379                         return -EINVAL;
 380                 }
 381                 if (radeon_check_and_fixup_offset(dev_priv, file_priv, &cmd[2])) {
 382                         DRM_ERROR("Invalid indx_buffer offset is %08X\n", cmd[2]);
 383                         return -EINVAL;
 384                 }
 385                 break;
 386
 387         case RADEON_CNTL_HOSTDATA_BLT:
 388         case RADEON_CNTL_PAINT_MULTI:
 389         case RADEON_CNTL_BITBLT_MULTI:
 390                 /* MSB of opcode: next DWORD GUI_CNTL */
 391                 if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
 392                               | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 393                         offset = cmd[2] << 10;
 394                         if (radeon_check_and_fixup_offset
 395                             (dev_priv, file_priv, &offset)) {
 396                                 DRM_ERROR("Invalid first packet offset\n");
 397                                 return -EINVAL;
 398                         }
 399                         cmd[2] = (cmd[2] & 0xffc00000) | offset >> 10;
 400                 }
 401
 402                 if ((cmd[1] & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
 403                     (cmd[1] & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 404                         offset = cmd[3] << 10;
 405                         if (radeon_check_and_fixup_offset
 406                             (dev_priv, file_priv, &offset)) {
 407                                 DRM_ERROR("Invalid second packet offset\n");
 408                                 return -EINVAL;
 409                         }
 410                         cmd[3] = (cmd[3] & 0xffc00000) | offset >> 10;
 411                 }
 412                 break;
 413
 414         default:
 415                 DRM_ERROR("Invalid packet type %x\n", cmd[0] & 0xff00);
 416                 return -EINVAL;
 417         }
 418
 419         return 0;
 420 }
 421
 422 /* ================================================================
 423  * CP hardware state programming functions
 424  */
 425
 426 static __inline__ void radeon_emit_clip_rect(drm_radeon_private_t * dev_priv,
 427                                              struct drm_clip_rect * box)
 428 {
 429         RING_LOCALS;
 430
 431         DRM_DEBUG("   box:  x1=%d y1=%d  x2=%d y2=%d\n",
 432                   box->x1, box->y1, box->x2, box->y2);
 433
 434         BEGIN_RING(4);
 435         OUT_RING(CP_PACKET0(RADEON_RE_TOP_LEFT, 0));
 436         OUT_RING((box->y1 << 16) | box->x1);
 437         OUT_RING(CP_PACKET0(RADEON_RE_WIDTH_HEIGHT, 0));
 438         OUT_RING(((box->y2 - 1) << 16) | (box->x2 - 1));
 439         ADVANCE_RING();
 440 }
 441
 442 /* Emit 1.1 state
 443  */
 444 static int radeon_emit_state(drm_radeon_private_t * dev_priv,
 445                              struct drm_file *file_priv,
 446                              drm_radeon_context_regs_t * ctx,
 447                              drm_radeon_texture_regs_t * tex,
 448                              unsigned int dirty)
 449 {
 450         RING_LOCALS;
 451         DRM_DEBUG("dirty=0x%08x\n", dirty);
 452
 453         if (dirty & RADEON_UPLOAD_CONTEXT) {
 454                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 455                                                   &ctx->rb3d_depthoffset)) {
 456                         DRM_ERROR("Invalid depth buffer offset\n");
 457                         return -EINVAL;
 458                 }
 459
 460                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 461                                                   &ctx->rb3d_coloroffset)) {
 462                         DRM_ERROR("Invalid depth buffer offset\n");
 463                         return -EINVAL;
 464                 }
 465
 466                 BEGIN_RING(14);
 467                 OUT_RING(CP_PACKET0(RADEON_PP_MISC, 6));
 468                 OUT_RING(ctx->pp_misc);
 469                 OUT_RING(ctx->pp_fog_color);
 470                 OUT_RING(ctx->re_solid_color);
 471                 OUT_RING(ctx->rb3d_blendcntl);
 472                 OUT_RING(ctx->rb3d_depthoffset);
 473                 OUT_RING(ctx->rb3d_depthpitch);
 474                 OUT_RING(ctx->rb3d_zstencilcntl);
 475                 OUT_RING(CP_PACKET0(RADEON_PP_CNTL, 2));
 476                 OUT_RING(ctx->pp_cntl);
 477                 OUT_RING(ctx->rb3d_cntl);
 478                 OUT_RING(ctx->rb3d_coloroffset);
 479                 OUT_RING(CP_PACKET0(RADEON_RB3D_COLORPITCH, 0));
 480                 OUT_RING(ctx->rb3d_colorpitch);
 481                 ADVANCE_RING();
 482         }
 483
 484         if (dirty & RADEON_UPLOAD_VERTFMT) {
 485                 BEGIN_RING(2);
 486                 OUT_RING(CP_PACKET0(RADEON_SE_COORD_FMT, 0));
 487                 OUT_RING(ctx->se_coord_fmt);
 488                 ADVANCE_RING();
 489         }
 490
 491         if (dirty & RADEON_UPLOAD_LINE) {
 492                 BEGIN_RING(5);
 493                 OUT_RING(CP_PACKET0(RADEON_RE_LINE_PATTERN, 1));
 494                 OUT_RING(ctx->re_line_pattern);
 495                 OUT_RING(ctx->re_line_state);
 496                 OUT_RING(CP_PACKET0(RADEON_SE_LINE_WIDTH, 0));
 497                 OUT_RING(ctx->se_line_width);
 498                 ADVANCE_RING();
 499         }
 500
 501         if (dirty & RADEON_UPLOAD_BUMPMAP) {
 502                 BEGIN_RING(5);
 503                 OUT_RING(CP_PACKET0(RADEON_PP_LUM_MATRIX, 0));
 504                 OUT_RING(ctx->pp_lum_matrix);
 505                 OUT_RING(CP_PACKET0(RADEON_PP_ROT_MATRIX_0, 1));
 506                 OUT_RING(ctx->pp_rot_matrix_0);
 507                 OUT_RING(ctx->pp_rot_matrix_1);
 508                 ADVANCE_RING();
 509         }
 510
 511         if (dirty & RADEON_UPLOAD_MASKS) {
 512                 BEGIN_RING(4);
 513                 OUT_RING(CP_PACKET0(RADEON_RB3D_STENCILREFMASK, 2));
 514                 OUT_RING(ctx->rb3d_stencilrefmask);
 515                 OUT_RING(ctx->rb3d_ropcntl);
 516                 OUT_RING(ctx->rb3d_planemask);
 517                 ADVANCE_RING();
 518         }
 519
 520         if (dirty & RADEON_UPLOAD_VIEWPORT) {
 521                 BEGIN_RING(7);
 522                 OUT_RING(CP_PACKET0(RADEON_SE_VPORT_XSCALE, 5));
 523                 OUT_RING(ctx->se_vport_xscale);
 524                 OUT_RING(ctx->se_vport_xoffset);
 525                 OUT_RING(ctx->se_vport_yscale);
 526                 OUT_RING(ctx->se_vport_yoffset);
 527                 OUT_RING(ctx->se_vport_zscale);
 528                 OUT_RING(ctx->se_vport_zoffset);
 529                 ADVANCE_RING();
 530         }
 531
 532         if (dirty & RADEON_UPLOAD_SETUP) {
 533                 BEGIN_RING(4);
 534                 OUT_RING(CP_PACKET0(RADEON_SE_CNTL, 0));
 535                 OUT_RING(ctx->se_cntl);
 536                 OUT_RING(CP_PACKET0(RADEON_SE_CNTL_STATUS, 0));
 537                 OUT_RING(ctx->se_cntl_status);
 538                 ADVANCE_RING();
 539         }
 540
 541         if (dirty & RADEON_UPLOAD_MISC) {
 542                 BEGIN_RING(2);
 543                 OUT_RING(CP_PACKET0(RADEON_RE_MISC, 0));
 544                 OUT_RING(ctx->re_misc);
 545                 ADVANCE_RING();
 546         }
 547
 548         if (dirty & RADEON_UPLOAD_TEX0) {
 549                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 550                                                   &tex[0].pp_txoffset)) {
 551                         DRM_ERROR("Invalid texture offset for unit 0\n");
 552                         return -EINVAL;
 553                 }
 554
 555                 BEGIN_RING(9);
 556                 OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_0, 5));
 557                 OUT_RING(tex[0].pp_txfilter);
 558                 OUT_RING(tex[0].pp_txformat);
 559                 OUT_RING(tex[0].pp_txoffset);
 560                 OUT_RING(tex[0].pp_txcblend);
 561                 OUT_RING(tex[0].pp_txablend);
 562                 OUT_RING(tex[0].pp_tfactor);
 563                 OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_0, 0));
 564                 OUT_RING(tex[0].pp_border_color);
 565                 ADVANCE_RING();
 566         }
 567
 568         if (dirty & RADEON_UPLOAD_TEX1) {
 569                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 570                                                   &tex[1].pp_txoffset)) {
 571                         DRM_ERROR("Invalid texture offset for unit 1\n");
 572                         return -EINVAL;
 573                 }
 574
 575                 BEGIN_RING(9);
 576                 OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_1, 5));
 577                 OUT_RING(tex[1].pp_txfilter);
 578                 OUT_RING(tex[1].pp_txformat);
 579                 OUT_RING(tex[1].pp_txoffset);
 580                 OUT_RING(tex[1].pp_txcblend);
 581                 OUT_RING(tex[1].pp_txablend);
 582                 OUT_RING(tex[1].pp_tfactor);
 583                 OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_1, 0));
 584                 OUT_RING(tex[1].pp_border_color);
 585                 ADVANCE_RING();
 586         }
 587
 588         if (dirty & RADEON_UPLOAD_TEX2) {
 589                 if (radeon_check_and_fixup_offset(dev_priv, file_priv,
 590                                                   &tex[2].pp_txoffset)) {
 591                         DRM_ERROR("Invalid texture offset for unit 2\n");
 592                         return -EINVAL;
 593                 }
 594
 595                 BEGIN_RING(9);
 596                 OUT_RING(CP_PACKET0(RADEON_PP_TXFILTER_2, 5));
 597                 OUT_RING(tex[2].pp_txfilter);
 598                 OUT_RING(tex[2].pp_txformat);
 599                 OUT_RING(tex[2].pp_txoffset);
 600                 OUT_RING(tex[2].pp_txcblend);
 601                 OUT_RING(tex[2].pp_txablend);
 602                 OUT_RING(tex[2].pp_tfactor);
 603                 OUT_RING(CP_PACKET0(RADEON_PP_BORDER_COLOR_2, 0));
 604                 OUT_RING(tex[2].pp_border_color);
 605                 ADVANCE_RING();
 606         }
 607
 608         return 0;
 609 }
 610
 611 /* Emit 1.2 state
 612  */
 613 static int radeon_emit_state2(drm_radeon_private_t * dev_priv,
 614                               struct drm_file *file_priv,
 615                               drm_radeon_state_t * state)
 616 {
 617         RING_LOCALS;
 618
 619         if (state->dirty & RADEON_UPLOAD_ZBIAS) {
 620                 BEGIN_RING(3);
 621                 OUT_RING(CP_PACKET0(RADEON_SE_ZBIAS_FACTOR, 1));
 622                 OUT_RING(state->context2.se_zbias_factor);
 623                 OUT_RING(state->context2.se_zbias_constant);
 624                 ADVANCE_RING();
 625         }
 626
 627         return radeon_emit_state(dev_priv, file_priv, &state->context,
 628                                  state->tex, state->dirty);
 629 }
 630
 631 /* New (1.3) state mechanism.  3 commands (packet, scalar, vector) in
 632  * 1.3 cmdbuffers allow all previous state to be updated as well as
 633  * the tcl scalar and vector areas.
 634  */
 635 static struct {
 636         int start;
 637         int len;
 638         const char *name;
 639 } packet[RADEON_MAX_STATE_PACKETS] = {
 640         {RADEON_PP_MISC, 7, "RADEON_PP_MISC"},
 641         {RADEON_PP_CNTL, 3, "RADEON_PP_CNTL"},
 642         {RADEON_RB3D_COLORPITCH, 1, "RADEON_RB3D_COLORPITCH"},
 643         {RADEON_RE_LINE_PATTERN, 2, "RADEON_RE_LINE_PATTERN"},
 644         {RADEON_SE_LINE_WIDTH, 1, "RADEON_SE_LINE_WIDTH"},
 645         {RADEON_PP_LUM_MATRIX, 1, "RADEON_PP_LUM_MATRIX"},
 646         {RADEON_PP_ROT_MATRIX_0, 2, "RADEON_PP_ROT_MATRIX_0"},
 647         {RADEON_RB3D_STENCILREFMASK, 3, "RADEON_RB3D_STENCILREFMASK"},
 648         {RADEON_SE_VPORT_XSCALE, 6, "RADEON_SE_VPORT_XSCALE"},
 649         {RADEON_SE_CNTL, 2, "RADEON_SE_CNTL"},
 650         {RADEON_SE_CNTL_STATUS, 1, "RADEON_SE_CNTL_STATUS"},
 651         {RADEON_RE_MISC, 1, "RADEON_RE_MISC"},
 652         {RADEON_PP_TXFILTER_0, 6, "RADEON_PP_TXFILTER_0"},
 653         {RADEON_PP_BORDER_COLOR_0, 1, "RADEON_PP_BORDER_COLOR_0"},
 654         {RADEON_PP_TXFILTER_1, 6, "RADEON_PP_TXFILTER_1"},
 655         {RADEON_PP_BORDER_COLOR_1, 1, "RADEON_PP_BORDER_COLOR_1"},
 656         {RADEON_PP_TXFILTER_2, 6, "RADEON_PP_TXFILTER_2"},
 657         {RADEON_PP_BORDER_COLOR_2, 1, "RADEON_PP_BORDER_COLOR_2"},
 658         {RADEON_SE_ZBIAS_FACTOR, 2, "RADEON_SE_ZBIAS_FACTOR"},
 659         {RADEON_SE_TCL_OUTPUT_VTX_FMT, 11, "RADEON_SE_TCL_OUTPUT_VTX_FMT"},
 660         {RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED, 17,
 661                     "RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED"},
 662         {R200_PP_TXCBLEND_0, 4, "R200_PP_TXCBLEND_0"},
 663         {R200_PP_TXCBLEND_1, 4, "R200_PP_TXCBLEND_1"},
 664         {R200_PP_TXCBLEND_2, 4, "R200_PP_TXCBLEND_2"},
 665         {R200_PP_TXCBLEND_3, 4, "R200_PP_TXCBLEND_3"},
 666         {R200_PP_TXCBLEND_4, 4, "R200_PP_TXCBLEND_4"},
 667         {R200_PP_TXCBLEND_5, 4, "R200_PP_TXCBLEND_5"},
 668         {R200_PP_TXCBLEND_6, 4, "R200_PP_TXCBLEND_6"},
 669         {R200_PP_TXCBLEND_7, 4, "R200_PP_TXCBLEND_7"},
 670         {R200_SE_TCL_LIGHT_MODEL_CTL_0, 6, "R200_SE_TCL_LIGHT_MODEL_CTL_0"},
 671         {R200_PP_TFACTOR_0, 6, "R200_PP_TFACTOR_0"},
 672         {R200_SE_VTX_FMT_0, 4, "R200_SE_VTX_FMT_0"},
 673         {R200_SE_VAP_CNTL, 1, "R200_SE_VAP_CNTL"},
 674         {R200_SE_TCL_MATRIX_SEL_0, 5, "R200_SE_TCL_MATRIX_SEL_0"},
 675         {R200_SE_TCL_TEX_PROC_CTL_2, 5, "R200_SE_TCL_TEX_PROC_CTL_2"},
 676         {R200_SE_TCL_UCP_VERT_BLEND_CTL, 1, "R200_SE_TCL_UCP_VERT_BLEND_CTL"},
 677         {R200_PP_TXFILTER_0, 6, "R200_PP_TXFILTER_0"},
 678         {R200_PP_TXFILTER_1, 6, "R200_PP_TXFILTER_1"},
 679         {R200_PP_TXFILTER_2, 6, "R200_PP_TXFILTER_2"},
 680         {R200_PP_TXFILTER_3, 6, "R200_PP_TXFILTER_3"},
 681         {R200_PP_TXFILTER_4, 6, "R200_PP_TXFILTER_4"},
 682         {R200_PP_TXFILTER_5, 6, "R200_PP_TXFILTER_5"},
 683         {R200_PP_TXOFFSET_0, 1, "R200_PP_TXOFFSET_0"},
 684         {R200_PP_TXOFFSET_1, 1, "R200_PP_TXOFFSET_1"},
 685         {R200_PP_TXOFFSET_2, 1, "R200_PP_TXOFFSET_2"},
 686         {R200_PP_TXOFFSET_3, 1, "R200_PP_TXOFFSET_3"},
 687         {R200_PP_TXOFFSET_4, 1, "R200_PP_TXOFFSET_4"},
 688         {R200_PP_TXOFFSET_5, 1, "R200_PP_TXOFFSET_5"},
 689         {R200_SE_VTE_CNTL, 1, "R200_SE_VTE_CNTL"},
 690         {R200_SE_TCL_OUTPUT_VTX_COMP_SEL, 1,
 691          "R200_SE_TCL_OUTPUT_VTX_COMP_SEL"},
 692         {R200_PP_TAM_DEBUG3, 1, "R200_PP_TAM_DEBUG3"},
 693         {R200_PP_CNTL_X, 1, "R200_PP_CNTL_X"},
 694         {R200_RB3D_DEPTHXY_OFFSET, 1, "R200_RB3D_DEPTHXY_OFFSET"},
 695         {R200_RE_AUX_SCISSOR_CNTL, 1, "R200_RE_AUX_SCISSOR_CNTL"},
 696         {R200_RE_SCISSOR_TL_0, 2, "R200_RE_SCISSOR_TL_0"},
 697         {R200_RE_SCISSOR_TL_1, 2, "R200_RE_SCISSOR_TL_1"},
 698         {R200_RE_SCISSOR_TL_2, 2, "R200_RE_SCISSOR_TL_2"},
 699         {R200_SE_VAP_CNTL_STATUS, 1, "R200_SE_VAP_CNTL_STATUS"},
 700         {R200_SE_VTX_STATE_CNTL, 1, "R200_SE_VTX_STATE_CNTL"},
 701         {R200_RE_POINTSIZE, 1, "R200_RE_POINTSIZE"},
 702         {R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0, 4,
 703                     "R200_SE_TCL_INPUT_VTX_VECTOR_ADDR_0"},
 704         {R200_PP_CUBIC_FACES_0, 1, "R200_PP_CUBIC_FACES_0"},    /* 61 */
 705         {R200_PP_CUBIC_OFFSET_F1_0, 5, "R200_PP_CUBIC_OFFSET_F1_0"}, /* 62 */
 706         {R200_PP_CUBIC_FACES_1, 1, "R200_PP_CUBIC_FACES_1"},
 707         {R200_PP_CUBIC_OFFSET_F1_1, 5, "R200_PP_CUBIC_OFFSET_F1_1"},
 708         {R200_PP_CUBIC_FACES_2, 1, "R200_PP_CUBIC_FACES_2"},
 709         {R200_PP_CUBIC_OFFSET_F1_2, 5, "R200_PP_CUBIC_OFFSET_F1_2"},
 710         {R200_PP_CUBIC_FACES_3, 1, "R200_PP_CUBIC_FACES_3"},
 711         {R200_PP_CUBIC_OFFSET_F1_3, 5, "R200_PP_CUBIC_OFFSET_F1_3"},
 712         {R200_PP_CUBIC_FACES_4, 1, "R200_PP_CUBIC_FACES_4"},
 713         {R200_PP_CUBIC_OFFSET_F1_4, 5, "R200_PP_CUBIC_OFFSET_F1_4"},
 714         {R200_PP_CUBIC_FACES_5, 1, "R200_PP_CUBIC_FACES_5"},
 715         {R200_PP_CUBIC_OFFSET_F1_5, 5, "R200_PP_CUBIC_OFFSET_F1_5"},
 716         {RADEON_PP_TEX_SIZE_0, 2, "RADEON_PP_TEX_SIZE_0"},
 717         {RADEON_PP_TEX_SIZE_1, 2, "RADEON_PP_TEX_SIZE_1"},
 718         {RADEON_PP_TEX_SIZE_2, 2, "RADEON_PP_TEX_SIZE_2"},
 719         {R200_RB3D_BLENDCOLOR, 3, "R200_RB3D_BLENDCOLOR"},
 720         {R200_SE_TCL_POINT_SPRITE_CNTL, 1, "R200_SE_TCL_POINT_SPRITE_CNTL"},
 721         {RADEON_PP_CUBIC_FACES_0, 1, "RADEON_PP_CUBIC_FACES_0"},
 722         {RADEON_PP_CUBIC_OFFSET_T0_0, 5, "RADEON_PP_CUBIC_OFFSET_T0_0"},
 723         {RADEON_PP_CUBIC_FACES_1, 1, "RADEON_PP_CUBIC_FACES_1"},
 724         {RADEON_PP_CUBIC_OFFSET_T1_0, 5, "RADEON_PP_CUBIC_OFFSET_T1_0"},
 725         {RADEON_PP_CUBIC_FACES_2, 1, "RADEON_PP_CUBIC_FACES_2"},
 726         {RADEON_PP_CUBIC_OFFSET_T2_0, 5, "RADEON_PP_CUBIC_OFFSET_T2_0"},
 727         {R200_PP_TRI_PERF, 2, "R200_PP_TRI_PERF"},
 728         {R200_PP_AFS_0, 32, "R200_PP_AFS_0"},     /* 85 */
 729         {R200_PP_AFS_1, 32, "R200_PP_AFS_1"},
 730         {R200_PP_TFACTOR_0, 8, "R200_ATF_TFACTOR"},
 731         {R200_PP_TXFILTER_0, 8, "R200_PP_TXCTLALL_0"},
 732         {R200_PP_TXFILTER_1, 8, "R200_PP_TXCTLALL_1"},
 733         {R200_PP_TXFILTER_2, 8, "R200_PP_TXCTLALL_2"},
 734         {R200_PP_TXFILTER_3, 8, "R200_PP_TXCTLALL_3"},
 735         {R200_PP_TXFILTER_4, 8, "R200_PP_TXCTLALL_4"},
 736         {R200_PP_TXFILTER_5, 8, "R200_PP_TXCTLALL_5"},
 737         {R200_VAP_PVS_CNTL_1, 2, "R200_VAP_PVS_CNTL"},
 738 };
 739
 740 /* ================================================================
 741  * Performance monitoring functions
 742  */
 743
 744 static void radeon_clear_box(drm_radeon_private_t * dev_priv,
 745                              int x, int y, int w, int h, int r, int g, int b)
 746 {
 747         u32 color;
 748         RING_LOCALS;
 749
 750         x += dev_priv->sarea_priv->boxes[0].x1;
 751         y += dev_priv->sarea_priv->boxes[0].y1;
 752
 753         switch (dev_priv->color_fmt) {
 754         case RADEON_COLOR_FORMAT_RGB565:
 755                 color = (((r & 0xf8) << 8) |
 756                          ((g & 0xfc) << 3) | ((b & 0xf8) >> 3));
 757                 break;
 758         case RADEON_COLOR_FORMAT_ARGB8888:
 759         default:
 760                 color = (((0xff) << 24) | (r << 16) | (g << 8) | b);
 761                 break;
 762         }
 763
 764         BEGIN_RING(4);
 765         RADEON_WAIT_UNTIL_3D_IDLE();
 766         OUT_RING(CP_PACKET0(RADEON_DP_WRITE_MASK, 0));
 767         OUT_RING(0xffffffff);
 768         ADVANCE_RING();
 769
 770         BEGIN_RING(6);
 771
 772         OUT_RING(CP_PACKET3(RADEON_CNTL_PAINT_MULTI, 4));
 773         OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 774                  RADEON_GMC_BRUSH_SOLID_COLOR |
 775                  (dev_priv->color_fmt << 8) |
 776                  RADEON_GMC_SRC_DATATYPE_COLOR |
 777                  RADEON_ROP3_P | RADEON_GMC_CLR_CMP_CNTL_DIS);
 778
 779         if (dev_priv->sarea_priv->pfCurrentPage == 1) {
 780                 OUT_RING(dev_priv->front_pitch_offset);
 781         } else {
 782                 OUT_RING(dev_priv->back_pitch_offset);
 783         }
 784
 785         OUT_RING(color);
 786
 787         OUT_RING((x << 16) | y);
 788         OUT_RING((w << 16) | h);
 789
 790         ADVANCE_RING();
 791 }
 792
 793 static void radeon_cp_performance_boxes(drm_radeon_private_t * dev_priv)
 794 {
 795         /* Collapse various things into a wait flag -- trying to
 796          * guess if userspase slept -- better just to have them tell us.
 797          */
 798         if (dev_priv->stats.last_frame_reads > 1 ||
 799             dev_priv->stats.last_clear_reads > dev_priv->stats.clears) {
 800                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 801         }
 802
 803         if (dev_priv->stats.freelist_loops) {
 804                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 805         }
 806
 807         /* Purple box for page flipping
 808          */
 809         if (dev_priv->stats.boxes & RADEON_BOX_FLIP)
 810                 radeon_clear_box(dev_priv, 4, 4, 8, 8, 255, 0, 255);
 811
 812         /* Red box if we have to wait for idle at any point
 813          */
 814         if (dev_priv->stats.boxes & RADEON_BOX_WAIT_IDLE)
 815                 radeon_clear_box(dev_priv, 16, 4, 8, 8, 255, 0, 0);
 816
 817         /* Blue box: lost context?
 818          */
 819
 820         /* Yellow box for texture swaps
 821          */
 822         if (dev_priv->stats.boxes & RADEON_BOX_TEXTURE_LOAD)
 823                 radeon_clear_box(dev_priv, 40, 4, 8, 8, 255, 255, 0);
 824
 825         /* Green box if hardware never idles (as far as we can tell)
 826          */
 827         if (!(dev_priv->stats.boxes & RADEON_BOX_DMA_IDLE))
 828                 radeon_clear_box(dev_priv, 64, 4, 8, 8, 0, 255, 0);
 829
 830         /* Draw bars indicating number of buffers allocated
 831          * (not a great measure, easily confused)
 832          */
 833         if (dev_priv->stats.requested_bufs) {
 834                 if (dev_priv->stats.requested_bufs > 100)
 835                         dev_priv->stats.requested_bufs = 100;
 836
 837                 radeon_clear_box(dev_priv, 4, 16,
 838                                  dev_priv->stats.requested_bufs, 4,
 839                                  196, 128, 128);
 840         }
 841
 842         memset(&dev_priv->stats, 0, sizeof(dev_priv->stats));
 843
 844 }
 845
 846 /* ================================================================
 847  * CP command dispatch functions
 848  */
 849
 850 static void radeon_cp_dispatch_clear(struct drm_device * dev,
 851                                      drm_radeon_clear_t * clear,
 852                                      drm_radeon_clear_rect_t * depth_boxes)
 853 {
 854         drm_radeon_private_t *dev_priv = dev->dev_private;
 855         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 856         drm_radeon_depth_clear_t *depth_clear = &dev_priv->depth_clear;
 857         int nbox = sarea_priv->nbox;
 858         struct drm_clip_rect *pbox = sarea_priv->boxes;
 859         unsigned int flags = clear->flags;
 860         u32 rb3d_cntl = 0, rb3d_stencilrefmask = 0;
 861         int i;
 862         RING_LOCALS;
 863         DRM_DEBUG("flags = 0x%x\n", flags);
 864
 865         dev_priv->stats.clears++;
 866
 867         if (dev_priv->sarea_priv->pfCurrentPage == 1) {
 868                 unsigned int tmp = flags;
 869
 870                 flags &= ~(RADEON_FRONT | RADEON_BACK);
 871                 if (tmp & RADEON_FRONT)
 872                         flags |= RADEON_BACK;
 873                 if (tmp & RADEON_BACK)
 874                         flags |= RADEON_FRONT;
 875         }
 876
 877         if (flags & (RADEON_FRONT | RADEON_BACK)) {
 878
 879                 BEGIN_RING(4);
 880
 881                 /* Ensure the 3D stream is idle before doing a
 882                  * 2D fill to clear the front or back buffer.
 883                  */
 884                 RADEON_WAIT_UNTIL_3D_IDLE();
 885
 886                 OUT_RING(CP_PACKET0(RADEON_DP_WRITE_MASK, 0));
 887                 OUT_RING(clear->color_mask);
 888
 889                 ADVANCE_RING();
 890
 891                 /* Make sure we restore the 3D state next time.
 892                  */
 893                 dev_priv->sarea_priv->ctx_owner = 0;
 894
 895                 for (i = 0; i < nbox; i++) {
 896                         int x = pbox[i].x1;
 897                         int y = pbox[i].y1;
 898                         int w = pbox[i].x2 - x;
 899                         int h = pbox[i].y2 - y;
 900
 901                         DRM_DEBUG("dispatch clear %d,%d-%d,%d flags 0x%x\n",
 902                                   x, y, w, h, flags);
 903
 904                         if (flags & RADEON_FRONT) {
 905                                 BEGIN_RING(6);
 906
 907                                 OUT_RING(CP_PACKET3
 908                                          (RADEON_CNTL_PAINT_MULTI, 4));
 909                                 OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 910                                          RADEON_GMC_BRUSH_SOLID_COLOR |
 911                                          (dev_priv->
 912                                           color_fmt << 8) |
 913                                          RADEON_GMC_SRC_DATATYPE_COLOR |
 914                                          RADEON_ROP3_P |
 915                                          RADEON_GMC_CLR_CMP_CNTL_DIS);
 916
 917                                 OUT_RING(dev_priv->front_pitch_offset);
 918                                 OUT_RING(clear->clear_color);
 919
 920                                 OUT_RING((x << 16) | y);
 921                                 OUT_RING((w << 16) | h);
 922
 923                                 ADVANCE_RING();
 924                         }
 925
 926                         if (flags & RADEON_BACK) {
 927                                 BEGIN_RING(6);
 928
 929                                 OUT_RING(CP_PACKET3
 930                                          (RADEON_CNTL_PAINT_MULTI, 4));
 931                                 OUT_RING(RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 932                                          RADEON_GMC_BRUSH_SOLID_COLOR |
 933                                          (dev_priv->
 934                                           color_fmt << 8) |
 935                                          RADEON_GMC_SRC_DATATYPE_COLOR |
 936                                          RADEON_ROP3_P |
 937                                          RADEON_GMC_CLR_CMP_CNTL_DIS);
 938
 939                                 OUT_RING(dev_priv->back_pitch_offset);
 940                                 OUT_RING(clear->clear_color);
 941
 942                                 OUT_RING((x << 16) | y);
 943                                 OUT_RING((w << 16) | h);
 944
 945                                 ADVANCE_RING();
 946                         }
 947                 }
 948         }
 949
 950         /* hyper z clear */
 951         /* no docs available, based on reverse engeneering by Stephane Marchesin */
 952         if ((flags & (RADEON_DEPTH | RADEON_STENCIL))
 953             && (flags & RADEON_CLEAR_FASTZ)) {
 954
 955                 int i;
 956                 int depthpixperline =
 957                     dev_priv->depth_fmt ==
 958                     RADEON_DEPTH_FORMAT_16BIT_INT_Z ? (dev_priv->depth_pitch /
 959                                                        2) : (dev_priv->
 960                                                              depth_pitch / 4);
 961
 962                 u32 clearmask;
 963
 964                 u32 tempRB3D_DEPTHCLEARVALUE = clear->clear_depth |
 965                     ((clear->depth_mask & 0xff) << 24);
 966
 967                 /* Make sure we restore the 3D state next time.
 968                  * we haven't touched any "normal" state - still need this?
 969                  */
 970                 dev_priv->sarea_priv->ctx_owner = 0;
 971
 972                 if ((dev_priv->flags & RADEON_HAS_HIERZ)
 973                     && (flags & RADEON_USE_HIERZ)) {
 974                         /* FIXME : reverse engineer that for Rx00 cards */
 975                         /* FIXME : the mask supposedly contains low-res z values. So can't set
 976                            just to the max (0xff? or actually 0x3fff?), need to take z clear
 977                            value into account? */
 978                         /* pattern seems to work for r100, though get slight
 979                            rendering errors with glxgears. If hierz is not enabled for r100,
 980                            only 4 bits which indicate clear (15,16,31,32, all zero) matter, the
 981                            other ones are ignored, and the same clear mask can be used. That's
 982                            very different behaviour than R200 which needs different clear mask
 983                            and different number of tiles to clear if hierz is enabled or not !?!
 984                          */
 985                         clearmask = (0xff << 22) | (0xff << 6) | 0x003f003f;
 986                 } else {
 987                         /* clear mask : chooses the clearing pattern.
 988                            rv250: could be used to clear only parts of macrotiles
 989                            (but that would get really complicated...)?
 990                            bit 0 and 1 (either or both of them ?!?!) are used to
 991                            not clear tile (or maybe one of the bits indicates if the tile is
 992                            compressed or not), bit 2 and 3 to not clear tile 1,...,.
 993                            Pattern is as follows:
 994                            | 0,1 | 4,5 | 8,9 |12,13|16,17|20,21|24,25|28,29|
 995                            bits -------------------------------------------------
 996                            | 2,3 | 6,7 |10,11|14,15|18,19|22,23|26,27|30,31|
 997                            rv100: clearmask covers 2x8 4x1 tiles, but one clear still
 998                            covers 256 pixels ?!?
 999                          */
1000                         clearmask = 0x0;
1001                 }
1002
1003                 BEGIN_RING(8);
1004                 RADEON_WAIT_UNTIL_2D_IDLE();
1005                 OUT_RING_REG(RADEON_RB3D_DEPTHCLEARVALUE,
1006                              tempRB3D_DEPTHCLEARVALUE);
1007                 /* what offset is this exactly ? */
1008                 OUT_RING_REG(RADEON_RB3D_ZMASKOFFSET, 0);
1009                 /* need ctlstat, otherwise get some strange black flickering */
1010                 OUT_RING_REG(RADEON_RB3D_ZCACHE_CTLSTAT,
1011                              RADEON_RB3D_ZC_FLUSH_ALL);
1012                 ADVANCE_RING();
1013
1014                 for (i = 0; i < nbox; i++) {
1015                         int tileoffset, nrtilesx, nrtilesy, j;
1016                         /* it looks like r200 needs rv-style clears, at least if hierz is not enabled? */
1017                         if ((dev_priv->flags & RADEON_HAS_HIERZ)
1018                             && !(dev_priv->microcode_version == UCODE_R200)) {
1019                                 /* FIXME : figure this out for r200 (when hierz is enabled). Or
1020                                    maybe r200 actually doesn't need to put the low-res z value into
1021                                    the tile cache like r100, but just needs to clear the hi-level z-buffer?
1022                                    Works for R100, both with hierz and without.
1023                                    R100 seems to operate on 2x1 8x8 tiles, but...
1024                                    odd: offset/nrtiles need to be 64 pix (4 block) aligned? Potentially
1025                                    problematic with resolutions which are not 64 pix aligned? */
1026                                 tileoffset =
1027                                     ((pbox[i].y1 >> 3) * depthpixperline +
1028                                      pbox[i].x1) >> 6;
1029                                 nrtilesx =
1030                                     ((pbox[i].x2 & ~63) -
1031                                      (pbox[i].x1 & ~63)) >> 4;
1032                                 nrtilesy =
1033                                     (pbox[i].y2 >> 3) - (pbox[i].y1 >> 3);
1034                                 for (j = 0; j <= nrtilesy; j++) {
1035                                         BEGIN_RING(4);
1036                                         OUT_RING(CP_PACKET3
1037                                                  (RADEON_3D_CLEAR_ZMASK, 2));
1038                                         /* first tile */
1039                                         OUT_RING(tileoffset * 8);
1040                                         /* the number of tiles to clear */
1041                                         OUT_RING(nrtilesx + 4);
1042                                         /* clear mask : chooses the clearing pattern. */
1043                                         OUT_RING(clearmask);
1044                                         ADVANCE_RING();
1045                                         tileoffset += depthpixperline >> 6;
1046                                 }
1047                         } else if (dev_priv->microcode_version == UCODE_R200) {
1048                                 /* works for rv250. */
1049                                 /* find first macro tile (8x2 4x4 z-pixels on rv250) */
1050                                 tileoffset =
1051                                     ((pbox[i].y1 >> 3) * depthpixperline +
1052                                      pbox[i].x1) >> 5;
1053                                 nrtilesx =
1054                                     (pbox[i].x2 >> 5) - (pbox[i].x1 >> 5);
1055                                 nrtilesy =
1056                                     (pbox[i].y2 >> 3) - (pbox[i].y1 >> 3);
1057                                 for (j = 0; j <= nrtilesy; j++) {
1058                                         BEGIN_RING(4);
1059                                         OUT_RING(CP_PACKET3
1060                                                  (RADEON_3D_CLEAR_ZMASK, 2));
1061                                         /* first tile */
1062                                         /* judging by the first tile offset needed, could possibly
1063                                            directly address/clear 4x4 tiles instead of 8x2 * 4x4
1064                                            macro tiles, though would still need clear mask for
1065                                            right/bottom if truely 4x4 granularity is desired ? */
1066                                         OUT_RING(tileoffset * 16);
1067                                         /* the number of tiles to clear */
1068                                         OUT_RING(nrtilesx + 1);
1069                                         /* clear mask : chooses the clearing pattern. */
1070                                         OUT_RING(clearmask);
1071                                         ADVANCE_RING();
1072                                         tileoffset += depthpixperline >> 5;
1073                                 }
1074                         } else {        /* rv 100 */
1075                                 /* rv100 might not need 64 pix alignment, who knows */
1076                                 /* offsets are, hmm, weird */
1077                                 tileoffset =
1078                                     ((pbox[i].y1 >> 4) * depthpixperline +
1079                                      pbox[i].x1) >> 6;
1080                                 nrtilesx =
1081                                     ((pbox[i].x2 & ~63) -
1082                                      (pbox[i].x1 & ~63)) >> 4;
1083                                 nrtilesy =
1084                                     (pbox[i].y2 >> 4) - (pbox[i].y1 >> 4);
1085                                 for (j = 0; j <= nrtilesy; j++) {
1086                                         BEGIN_RING(4);
1087                                         OUT_RING(CP_PACKET3
1088                                                  (RADEON_3D_CLEAR_ZMASK, 2));
1089                                         OUT_RING(tileoffset * 128);
1090                                         /* the number of tiles to clear */
1091                                         OUT_RING(nrtilesx + 4);
1092                                         /* clear mask : chooses the clearing pattern. */
1093                                         OUT_RING(clearmask);
1094                                         ADVANCE_RING();
1095                                         tileoffset += depthpixperline >> 6;
1096                                 }
1097                         }
1098                 }
1099
1100                 /* TODO don't always clear all hi-level z tiles */
1101                 if ((dev_priv->flags & RADEON_HAS_HIERZ)
1102                     && (dev_priv->microcode_version == UCODE_R200)
1103                     && (flags & RADEON_USE_HIERZ))
1104                         /* r100 and cards without hierarchical z-buffer have no high-level z-buffer */
1105                         /* FIXME : the mask supposedly contains low-res z values. So can't set
1106                            just to the max (0xff? or actually 0x3fff?), need to take z clear
1107                            value into account? */
1108                 {
1109                         BEGIN_RING(4);
1110                         OUT_RING(CP_PACKET3(RADEON_3D_CLEAR_HIZ, 2));
1111                         OUT_RING(0x0);  /* First tile */
1112                         OUT_RING(0x3cc0);
1113                         OUT_RING((0xff << 22) | (0xff << 6) | 0x003f003f);
1114                         ADVANCE_RING();
1115                 }
1116         }
1117
1118         /* We have to clear the depth and/or stencil buffers by
1119          * rendering a quad into just those buffers.  Thus, we have to
1120          * make sure the 3D engine is configured correctly.
1121          */
1122         else if ((dev_priv->microcode_version == UCODE_R200) &&
1123                 (flags & (RADEON_DEPTH | RADEON_STENCIL))) {
1124
1125                 int tempPP_CNTL;
1126                 int tempRE_CNTL;
1127                 int tempRB3D_CNTL;
1128                 int tempRB3D_ZSTENCILCNTL;
1129                 int tempRB3D_STENCILREFMASK;
1130                 int tempRB3D_PLANEMASK;
1131                 int tempSE_CNTL;
1132                 int tempSE_VTE_CNTL;
1133                 int tempSE_VTX_FMT_0;
1134                 int tempSE_VTX_FMT_1;
1135                 int tempSE_VAP_CNTL;
1136                 int tempRE_AUX_SCISSOR_CNTL;
1137
1138                 tempPP_CNTL = 0;
1139                 tempRE_CNTL = 0;
1140
1141                 tempRB3D_CNTL = depth_clear->rb3d_cntl;
1142
1143                 tempRB3D_ZSTENCILCNTL = depth_clear->rb3d_zstencilcntl;
1144                 tempRB3D_STENCILREFMASK = 0x0;
1145
1146                 tempSE_CNTL = depth_clear->se_cntl;
1147
1148                 /* Disable TCL */
1149
1150                 tempSE_VAP_CNTL = (     /* SE_VAP_CNTL__FORCE_W_TO_ONE_MASK |  */
1151                                           (0x9 <<
1152                                            SE_VAP_CNTL__VF_MAX_VTX_NUM__SHIFT));
1153
1154                 tempRB3D_PLANEMASK = 0x0;
1155
1156                 tempRE_AUX_SCISSOR_CNTL = 0x0;
1157
1158                 tempSE_VTE_CNTL =
1159                     SE_VTE_CNTL__VTX_XY_FMT_MASK | SE_VTE_CNTL__VTX_Z_FMT_MASK;
1160
1161                 /* Vertex format (X, Y, Z, W) */
1162                 tempSE_VTX_FMT_0 =
1163                     SE_VTX_FMT_0__VTX_Z0_PRESENT_MASK |
1164                     SE_VTX_FMT_0__VTX_W0_PRESENT_MASK;
1165                 tempSE_VTX_FMT_1 = 0x0;
1166
1167                 /*
1168                  * Depth buffer specific enables
1169                  */
1170                 if (flags & RADEON_DEPTH) {
1171                         /* Enable depth buffer */
1172                         tempRB3D_CNTL |= RADEON_Z_ENABLE;
1173                 } else {
1174                         /* Disable depth buffer */
1175                         tempRB3D_CNTL &= ~RADEON_Z_ENABLE;
1176                 }
1177
1178                 /*
1179                  * Stencil buffer specific enables
1180                  */
1181                 if (flags & RADEON_STENCIL) {
1182                         tempRB3D_CNTL |= RADEON_STENCIL_ENABLE;
1183                         tempRB3D_STENCILREFMASK = clear->depth_mask;
1184                 } else {
1185                         tempRB3D_CNTL &= ~RADEON_STENCIL_ENABLE;
1186                         tempRB3D_STENCILREFMASK = 0x00000000;
1187                 }
1188
1189                 if (flags & RADEON_USE_COMP_ZBUF) {
1190                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_COMPRESSION_ENABLE |
1191                             RADEON_Z_DECOMPRESSION_ENABLE;
1192                 }
1193                 if (flags & RADEON_USE_HIERZ) {
1194                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_HIERARCHY_ENABLE;
1195                 }
1196
1197                 BEGIN_RING(26);
1198                 RADEON_WAIT_UNTIL_2D_IDLE();
1199
1200                 OUT_RING_REG(RADEON_PP_CNTL, tempPP_CNTL);
1201                 OUT_RING_REG(R200_RE_CNTL, tempRE_CNTL);
1202                 OUT_RING_REG(RADEON_RB3D_CNTL, tempRB3D_CNTL);
1203                 OUT_RING_REG(RADEON_RB3D_ZSTENCILCNTL, tempRB3D_ZSTENCILCNTL);
1204                 OUT_RING_REG(RADEON_RB3D_STENCILREFMASK,
1205                              tempRB3D_STENCILREFMASK);
1206                 OUT_RING_REG(RADEON_RB3D_PLANEMASK, tempRB3D_PLANEMASK);
1207                 OUT_RING_REG(RADEON_SE_CNTL, tempSE_CNTL);
1208                 OUT_RING_REG(R200_SE_VTE_CNTL, tempSE_VTE_CNTL);
1209                 OUT_RING_REG(R200_SE_VTX_FMT_0, tempSE_VTX_FMT_0);
1210                 OUT_RING_REG(R200_SE_VTX_FMT_1, tempSE_VTX_FMT_1);
1211                 OUT_RING_REG(R200_SE_VAP_CNTL, tempSE_VAP_CNTL);
1212                 OUT_RING_REG(R200_RE_AUX_SCISSOR_CNTL, tempRE_AUX_SCISSOR_CNTL);
1213                 ADVANCE_RING();
1214
1215                 /* Make sure we restore the 3D state next time.
1216                  */
1217                 dev_priv->sarea_priv->ctx_owner = 0;
1218
1219                 for (i = 0; i < nbox; i++) {
1220
1221                         /* Funny that this should be required --
1222                          *  sets top-left?
1223                          */
1224                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1225
1226                         BEGIN_RING(14);
1227                         OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 12));
1228                         OUT_RING((RADEON_PRIM_TYPE_RECT_LIST |
1229                                   RADEON_PRIM_WALK_RING |
1230                                   (3 << RADEON_NUM_VERTICES_SHIFT)));
1231                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1232                         OUT_RING(depth_boxes[i].ui[CLEAR_Y1]);
1233                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1234                         OUT_RING(0x3f800000);
1235                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1236                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1237                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1238                         OUT_RING(0x3f800000);
1239                         OUT_RING(depth_boxes[i].ui[CLEAR_X2]);
1240                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1241                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1242                         OUT_RING(0x3f800000);
1243                         ADVANCE_RING();
1244                 }
1245         } else if ((flags & (RADEON_DEPTH | RADEON_STENCIL))) {
1246
1247                 int tempRB3D_ZSTENCILCNTL = depth_clear->rb3d_zstencilcntl;
1248
1249                 rb3d_cntl = depth_clear->rb3d_cntl;
1250
1251                 if (flags & RADEON_DEPTH) {
1252                         rb3d_cntl |= RADEON_Z_ENABLE;
1253                 } else {
1254                         rb3d_cntl &= ~RADEON_Z_ENABLE;
1255                 }
1256
1257                 if (flags & RADEON_STENCIL) {
1258                         rb3d_cntl |= RADEON_STENCIL_ENABLE;
1259                         rb3d_stencilrefmask = clear->depth_mask;        /* misnamed field */
1260                 } else {
1261                         rb3d_cntl &= ~RADEON_STENCIL_ENABLE;
1262                         rb3d_stencilrefmask = 0x00000000;
1263                 }
1264
1265                 if (flags & RADEON_USE_COMP_ZBUF) {
1266                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_COMPRESSION_ENABLE |
1267                             RADEON_Z_DECOMPRESSION_ENABLE;
1268                 }
1269                 if (flags & RADEON_USE_HIERZ) {
1270                         tempRB3D_ZSTENCILCNTL |= RADEON_Z_HIERARCHY_ENABLE;
1271                 }
1272
1273                 BEGIN_RING(13);
1274                 RADEON_WAIT_UNTIL_2D_IDLE();
1275
1276                 OUT_RING(CP_PACKET0(RADEON_PP_CNTL, 1));
1277                 OUT_RING(0x00000000);
1278                 OUT_RING(rb3d_cntl);
1279
1280                 OUT_RING_REG(RADEON_RB3D_ZSTENCILCNTL, tempRB3D_ZSTENCILCNTL);
1281                 OUT_RING_REG(RADEON_RB3D_STENCILREFMASK, rb3d_stencilrefmask);
1282                 OUT_RING_REG(RADEON_RB3D_PLANEMASK, 0x00000000);
1283                 OUT_RING_REG(RADEON_SE_CNTL, depth_clear->se_cntl);
1284                 ADVANCE_RING();
1285
1286                 /* Make sure we restore the 3D state next time.
1287                  */
1288                 dev_priv->sarea_priv->ctx_owner = 0;
1289
1290                 for (i = 0; i < nbox; i++) {
1291
1292                         /* Funny that this should be required --
1293                          *  sets top-left?
1294                          */
1295                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1296
1297                         BEGIN_RING(15);
1298
1299                         OUT_RING(CP_PACKET3(RADEON_3D_DRAW_IMMD, 13));
1300                         OUT_RING(RADEON_VTX_Z_PRESENT |
1301                                  RADEON_VTX_PKCOLOR_PRESENT);
1302                         OUT_RING((RADEON_PRIM_TYPE_RECT_LIST |
1303                                   RADEON_PRIM_WALK_RING |
1304                                   RADEON_MAOS_ENABLE |
1305                                   RADEON_VTX_FMT_RADEON_MODE |
1306                                   (3 << RADEON_NUM_VERTICES_SHIFT)));
1307
1308                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1309                         OUT_RING(depth_boxes[i].ui[CLEAR_Y1]);
1310                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1311                         OUT_RING(0x0);
1312
1313                         OUT_RING(depth_boxes[i].ui[CLEAR_X1]);
1314                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1315                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1316                         OUT_RING(0x0);
1317
1318                         OUT_RING(depth_boxes[i].ui[CLEAR_X2]);
1319                         OUT_RING(depth_boxes[i].ui[CLEAR_Y2]);
1320                         OUT_RING(depth_boxes[i].ui[CLEAR_DEPTH]);
1321                         OUT_RING(0x0);
1322
1323                         ADVANCE_RING();
1324                 }
1325         }
1326
1327         /* Increment the clear counter.  The client-side 3D driver must
1328          * wait on this value before performing the clear ioctl.  We
1329          * need this because the card's so damned fast...
1330          */
1331         dev_priv->sarea_priv->last_clear++;
1332
1333         BEGIN_RING(4);
1334
1335         RADEON_CLEAR_AGE(dev_priv->sarea_priv->last_clear);
1336         RADEON_WAIT_UNTIL_IDLE();
1337
1338         ADVANCE_RING();
1339 }
1340
1341 static void radeon_cp_dispatch_swap(struct drm_device * dev)
1342 {
1343         drm_radeon_private_t *dev_priv = dev->dev_private;
1344         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1345         int nbox = sarea_priv->nbox;
1346         struct drm_clip_rect *pbox = sarea_priv->boxes;
1347         int i;
1348         RING_LOCALS;
1349         DRM_DEBUG("\n");
1350
1351         /* Do some trivial performance monitoring...
1352          */
1353         if (dev_priv->do_boxes)
1354                 radeon_cp_performance_boxes(dev_priv);
1355
1356         /* Wait for the 3D stream to idle before dispatching the bitblt.
1357          * This will prevent data corruption between the two streams.
1358          */
1359         BEGIN_RING(2);
1360
1361         RADEON_WAIT_UNTIL_3D_IDLE();
1362
1363         ADVANCE_RING();
1364
1365         for (i = 0; i < nbox; i++) {
1366                 int x = pbox[i].x1;
1367                 int y = pbox[i].y1;
1368                 int w = pbox[i].x2 - x;
1369                 int h = pbox[i].y2 - y;
1370
1371                 DRM_DEBUG("dispatch swap %d,%d-%d,%d\n", x, y, w, h);
1372
1373                 BEGIN_RING(9);
1374
1375                 OUT_RING(CP_PACKET0(RADEON_DP_GUI_MASTER_CNTL, 0));
1376                 OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
1377                          RADEON_GMC_DST_PITCH_OFFSET_CNTL |
1378                          RADEON_GMC_BRUSH_NONE |
1379                          (dev_priv->color_fmt << 8) |
1380                          RADEON_GMC_SRC_DATATYPE_COLOR |
1381                          RADEON_ROP3_S |
1382                          RADEON_DP_SRC_SOURCE_MEMORY |
1383                          RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
1384
1385                 /* Make this work even if front & back are flipped:
1386                  */
1387                 OUT_RING(CP_PACKET0(RADEON_SRC_PITCH_OFFSET, 1));
1388                 if (dev_priv->sarea_priv->pfCurrentPage == 0) {
1389                         OUT_RING(dev_priv->back_pitch_offset);
1390                         OUT_RING(dev_priv->front_pitch_offset);
1391                 } else {
1392                         OUT_RING(dev_priv->front_pitch_offset);
1393                         OUT_RING(dev_priv->back_pitch_offset);
1394                 }
1395
1396                 OUT_RING(CP_PACKET0(RADEON_SRC_X_Y, 2));
1397                 OUT_RING((x << 16) | y);
1398                 OUT_RING((x << 16) | y);
1399                 OUT_RING((w << 16) | h);
1400
1401                 ADVANCE_RING();
1402         }
1403
1404         /* Increment the frame counter.  The client-side 3D driver must
1405          * throttle the framerate by waiting for this value before
1406          * performing the swapbuffer ioctl.
1407          */
1408         dev_priv->sarea_priv->last_frame++;
1409
1410         BEGIN_RING(4);
1411
1412         RADEON_FRAME_AGE(dev_priv->sarea_priv->last_frame);
1413         RADEON_WAIT_UNTIL_2D_IDLE();
1414
1415         ADVANCE_RING();
1416 }
1417
1418 static void radeon_cp_dispatch_flip(struct drm_device * dev)
1419 {
1420         drm_radeon_private_t *dev_priv = dev->dev_private;
1421         struct drm_sarea *sarea = (struct drm_sarea *) dev_priv->sarea->handle;
1422         int offset = (dev_priv->sarea_priv->pfCurrentPage == 1)
1423             ? dev_priv->front_offset : dev_priv->back_offset;
1424         RING_LOCALS;
1425         DRM_DEBUG("%s: pfCurrentPage=%d\n",
1426                   __FUNCTION__,
1427                   dev_priv->sarea_priv->pfCurrentPage);
1428
1429         /* Do some trivial performance monitoring...
1430          */
1431         if (dev_priv->do_boxes) {
1432                 dev_priv->stats.boxes |= RADEON_BOX_FLIP;
1433                 radeon_cp_performance_boxes(dev_priv);
1434         }
1435
1436         /* Update the frame offsets for both CRTCs
1437          */
1438         BEGIN_RING(6);
1439
1440         RADEON_WAIT_UNTIL_3D_IDLE();
1441         OUT_RING_REG(RADEON_CRTC_OFFSET,
1442                      ((sarea->frame.y * dev_priv->front_pitch +
1443                        sarea->frame.x * (dev_priv->color_fmt - 2)) & ~7)
1444                      + offset);
1445         OUT_RING_REG(RADEON_CRTC2_OFFSET, dev_priv->sarea_priv->crtc2_base
1446                      + offset);
1447
1448         ADVANCE_RING();
1449
1450         /* Increment the frame counter.  The client-side 3D driver must
1451          * throttle the framerate by waiting for this value before
1452          * performing the swapbuffer ioctl.
1453          */
1454         dev_priv->sarea_priv->last_frame++;
1455         dev_priv->sarea_priv->pfCurrentPage =
1456                 1 - dev_priv->sarea_priv->pfCurrentPage;
1457
1458         BEGIN_RING(2);
1459
1460         RADEON_FRAME_AGE(dev_priv->sarea_priv->last_frame);
1461
1462         ADVANCE_RING();
1463 }
1464
1465 static int bad_prim_vertex_nr(int primitive, int nr)
1466 {
1467         switch (primitive & RADEON_PRIM_TYPE_MASK) {
1468         case RADEON_PRIM_TYPE_NONE:
1469         case RADEON_PRIM_TYPE_POINT:
1470                 return nr < 1;
1471         case RADEON_PRIM_TYPE_LINE:
1472                 return (nr & 1) || nr == 0;
1473         case RADEON_PRIM_TYPE_LINE_STRIP:
1474                 return nr < 2;
1475         case RADEON_PRIM_TYPE_TRI_LIST:
1476         case RADEON_PRIM_TYPE_3VRT_POINT_LIST:
1477         case RADEON_PRIM_TYPE_3VRT_LINE_LIST:
1478         case RADEON_PRIM_TYPE_RECT_LIST:
1479                 return nr % 3 || nr == 0;
1480         case RADEON_PRIM_TYPE_TRI_FAN:
1481         case RADEON_PRIM_TYPE_TRI_STRIP:
1482                 return nr < 3;
1483         default:
1484                 return 1;
1485         }
1486 }
1487
1488 typedef struct {
1489         unsigned int start;
1490         unsigned int finish;
1491         unsigned int prim;
1492         unsigned int numverts;
1493         unsigned int offset;
1494         unsigned int vc_format;
1495 } drm_radeon_tcl_prim_t;
1496
1497 static void radeon_cp_dispatch_vertex(struct drm_device * dev,
1498                                       struct drm_buf * buf,
1499                                       drm_radeon_tcl_prim_t * prim)
1500 {
1501         drm_radeon_private_t *dev_priv = dev->dev_private;
1502         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1503         int offset = dev_priv->gart_buffers_offset + buf->offset + prim->start;
1504         int numverts = (int)prim->numverts;
1505         int nbox = sarea_priv->nbox;
1506         int i = 0;
1507         RING_LOCALS;
1508
1509         DRM_DEBUG("hwprim 0x%x vfmt 0x%x %d..%d %d verts\n",
1510                   prim->prim,
1511                   prim->vc_format, prim->start, prim->finish, prim->numverts);
1512
1513         if (bad_prim_vertex_nr(prim->prim, prim->numverts)) {
1514                 DRM_ERROR("bad prim %x numverts %d\n",
1515                           prim->prim, prim->numverts);
1516                 return;
1517         }
1518
1519         do {
1520                 /* Emit the next cliprect */
1521                 if (i < nbox) {
1522                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1523                 }
1524
1525                 /* Emit the vertex buffer rendering commands */
1526                 BEGIN_RING(5);
1527
1528                 OUT_RING(CP_PACKET3(RADEON_3D_RNDR_GEN_INDX_PRIM, 3));
1529                 OUT_RING(offset);
1530                 OUT_RING(numverts);
1531                 OUT_RING(prim->vc_format);
1532                 OUT_RING(prim->prim | RADEON_PRIM_WALK_LIST |
1533                          RADEON_COLOR_ORDER_RGBA |
1534                          RADEON_VTX_FMT_RADEON_MODE |
1535                          (numverts << RADEON_NUM_VERTICES_SHIFT));
1536
1537                 ADVANCE_RING();
1538
1539                 i++;
1540         } while (i < nbox);
1541 }
1542
1543 static void radeon_cp_discard_buffer(struct drm_device * dev, struct drm_buf * buf)
1544 {
1545         drm_radeon_private_t *dev_priv = dev->dev_private;
1546         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1547         RING_LOCALS;
1548
1549         buf_priv->age = ++dev_priv->sarea_priv->last_dispatch;
1550
1551         /* Emit the vertex buffer age */
1552         BEGIN_RING(2);
1553         RADEON_DISPATCH_AGE(buf_priv->age);
1554         ADVANCE_RING();
1555
1556         buf->pending = 1;
1557         buf->used = 0;
1558 }
1559
1560 static void radeon_cp_dispatch_indirect(struct drm_device * dev,
1561                                         struct drm_buf * buf, int start, int end)
1562 {
1563         drm_radeon_private_t *dev_priv = dev->dev_private;
1564         RING_LOCALS;
1565         DRM_DEBUG("indirect: buf=%d s=0x%x e=0x%x\n", buf->idx, start, end);
1566
1567         if (start != end) {
1568                 int offset = (dev_priv->gart_buffers_offset
1569                               + buf->offset + start);
1570                 int dwords = (end - start + 3) / sizeof(u32);
1571
1572                 /* Indirect buffer data must be an even number of
1573                  * dwords, so if we've been given an odd number we must
1574                  * pad the data with a Type-2 CP packet.
1575                  */
1576                 if (dwords & 1) {
1577                         u32 *data = (u32 *)
1578                             ((char *)dev->agp_buffer_map->handle
1579                              + buf->offset + start);
1580                         data[dwords++] = RADEON_CP_PACKET2;
1581                 }
1582
1583                 /* Fire off the indirect buffer */
1584                 BEGIN_RING(3);
1585
1586                 OUT_RING(CP_PACKET0(RADEON_CP_IB_BASE, 1));
1587                 OUT_RING(offset);
1588                 OUT_RING(dwords);
1589
1590                 ADVANCE_RING();
1591         }
1592 }
1593
1594 static void radeon_cp_dispatch_indices(struct drm_device * dev,
1595                                        struct drm_buf * elt_buf,
1596                                        drm_radeon_tcl_prim_t * prim)
1597 {
1598         drm_radeon_private_t *dev_priv = dev->dev_private;
1599         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1600         int offset = dev_priv->gart_buffers_offset + prim->offset;
1601         u32 *data;
1602         int dwords;
1603         int i = 0;
1604         int start = prim->start + RADEON_INDEX_PRIM_OFFSET;
1605         int count = (prim->finish - start) / sizeof(u16);
1606         int nbox = sarea_priv->nbox;
1607
1608         DRM_DEBUG("hwprim 0x%x vfmt 0x%x %d..%d offset: %x nr %d\n",
1609                   prim->prim,
1610                   prim->vc_format,
1611                   prim->start, prim->finish, prim->offset, prim->numverts);
1612
1613         if (bad_prim_vertex_nr(prim->prim, count)) {
1614                 DRM_ERROR("bad prim %x count %d\n", prim->prim, count);
1615                 return;
1616         }
1617
1618         if (start >= prim->finish || (prim->start & 0x7)) {
1619                 DRM_ERROR("buffer prim %d\n", prim->prim);
1620                 return;
1621         }
1622
1623         dwords = (prim->finish - prim->start + 3) / sizeof(u32);
1624
1625         data = (u32 *) ((char *)dev->agp_buffer_map->handle +
1626                         elt_buf->offset + prim->start);
1627
1628         data[0] = CP_PACKET3(RADEON_3D_RNDR_GEN_INDX_PRIM, dwords - 2);
1629         data[1] = offset;
1630         data[2] = prim->numverts;
1631         data[3] = prim->vc_format;
1632         data[4] = (prim->prim |
1633                    RADEON_PRIM_WALK_IND |
1634                    RADEON_COLOR_ORDER_RGBA |
1635                    RADEON_VTX_FMT_RADEON_MODE |
1636                    (count << RADEON_NUM_VERTICES_SHIFT));
1637
1638         do {
1639                 if (i < nbox)
1640                         radeon_emit_clip_rect(dev_priv, &sarea_priv->boxes[i]);
1641
1642                 radeon_cp_dispatch_indirect(dev, elt_buf,
1643                                             prim->start, prim->finish);
1644
1645                 i++;
1646         } while (i < nbox);
1647
1648 }
1649
1650 #define RADEON_MAX_TEXTURE_SIZE RADEON_BUFFER_SIZE
1651
1652 static int radeon_cp_dispatch_texture(struct drm_device * dev,
1653                                       struct drm_file *file_priv,
1654                                       drm_radeon_texture_t * tex,
1655                                       drm_radeon_tex_image_t * image)
1656 {
1657         drm_radeon_private_t *dev_priv = dev->dev_private;
1658         struct drm_buf *buf;
1659         u32 format;
1660         u32 *buffer;
1661         const u8 __user *data;
1662         int size, dwords, tex_width, blit_width, spitch;
1663         u32 height;
1664         int i;
1665         u32 texpitch, microtile;
1666         u32 offset;
1667         RING_LOCALS;
1668
1669         if (radeon_check_and_fixup_offset(dev_priv, file_priv, &tex->offset)) {
1670                 DRM_ERROR("Invalid destination offset\n");
1671                 return -EINVAL;
1672         }
1673
1674         dev_priv->stats.boxes |= RADEON_BOX_TEXTURE_LOAD;
1675
1676         /* Flush the pixel cache.  This ensures no pixel data gets mixed
1677          * up with the texture data from the host data blit, otherwise
1678          * part of the texture image may be corrupted.
1679          */
1680         BEGIN_RING(4);
1681         RADEON_FLUSH_CACHE();
1682         RADEON_WAIT_UNTIL_IDLE();
1683         ADVANCE_RING();
1684
1685         /* The compiler won't optimize away a division by a variable,
1686          * even if the only legal values are powers of two.  Thus, we'll
1687          * use a shift instead.
1688          */
1689         switch (tex->format) {
1690         case RADEON_TXFORMAT_ARGB8888:
1691         case RADEON_TXFORMAT_RGBA8888:
1692                 format = RADEON_COLOR_FORMAT_ARGB8888;
1693                 tex_width = tex->width * 4;
1694                 blit_width = image->width * 4;
1695                 break;
1696         case RADEON_TXFORMAT_AI88:
1697         case RADEON_TXFORMAT_ARGB1555:
1698         case RADEON_TXFORMAT_RGB565:
1699         case RADEON_TXFORMAT_ARGB4444:
1700         case RADEON_TXFORMAT_VYUY422:
1701         case RADEON_TXFORMAT_YVYU422:
1702                 format = RADEON_COLOR_FORMAT_RGB565;
1703                 tex_width = tex->width * 2;
1704                 blit_width = image->width * 2;
1705                 break;
1706         case RADEON_TXFORMAT_I8:
1707         case RADEON_TXFORMAT_RGB332:
1708                 format = RADEON_COLOR_FORMAT_CI8;
1709                 tex_width = tex->width * 1;
1710                 blit_width = image->width * 1;
1711                 break;
1712         default:
1713                 DRM_ERROR("invalid texture format %d\n", tex->format);
1714                 return -EINVAL;
1715         }
1716         spitch = blit_width >> 6;
1717         if (spitch == 0 && image->height > 1)
1718                 return -EINVAL;
1719
1720         texpitch = tex->pitch;
1721         if ((texpitch << 22) & RADEON_DST_TILE_MICRO) {
1722                 microtile = 1;
1723                 if (tex_width < 64) {
1724                         texpitch &= ~(RADEON_DST_TILE_MICRO >> 22);
1725                         /* we got tiled coordinates, untile them */
1726                         image->x *= 2;
1727                 }
1728         } else
1729                 microtile = 0;
1730
1731         DRM_DEBUG("tex=%dx%d blit=%d\n", tex_width, tex->height, blit_width);
1732
1733         do {
1734                 DRM_DEBUG("tex: ofs=0x%x p=%d f=%d x=%hd y=%hd w=%hd h=%hd\n",
1735                           tex->offset >> 10, tex->pitch, tex->format,
1736                           image->x, image->y, image->width, image->height);
1737
1738                 /* Make a copy of some parameters in case we have to
1739                  * update them for a multi-pass texture blit.
1740                  */
1741                 height = image->height;
1742                 data = (const u8 __user *)image->data;
1743
1744                 size = height * blit_width;
1745
1746                 if (size > RADEON_MAX_TEXTURE_SIZE) {
1747                         height = RADEON_MAX_TEXTURE_SIZE / blit_width;
1748                         size = height * blit_width;
1749                 } else if (size < 4 && size > 0) {
1750                         size = 4;
1751                 } else if (size == 0) {
1752                         return 0;
1753                 }
1754
1755                 buf = radeon_freelist_get(dev);
1756                 if (0 && !buf) {
1757                         radeon_do_cp_idle(dev_priv);
1758                         buf = radeon_freelist_get(dev);
1759                 }
1760                 if (!buf) {
1761                         DRM_DEBUG("radeon_cp_dispatch_texture: EAGAIN\n");
1762                         if (DRM_COPY_TO_USER(tex->image, image, sizeof(*image)))
1763                                 return -EFAULT;
1764                         return -EAGAIN;
1765                 }
1766
1767                 /* Dispatch the indirect buffer.
1768                  */
1769                 buffer =
1770                     (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
1771                 dwords = size / 4;
1772
1773 #define RADEON_COPY_MT(_buf, _data, _width) \
1774         do { \
1775                 if (DRM_COPY_FROM_USER(_buf, _data, (_width))) {\
1776                         DRM_ERROR("EFAULT on pad, %d bytes\n", (_width)); \
1777                         return -EFAULT; \
1778                 } \
1779         } while(0)
1780
1781                 if (microtile) {
1782                         /* texture micro tiling in use, minimum texture width is thus 16 bytes.
1783                            however, we cannot use blitter directly for texture width < 64 bytes,
1784                            since minimum tex pitch is 64 bytes and we need this to match
1785                            the texture width, otherwise the blitter will tile it wrong.
1786                            Thus, tiling manually in this case. Additionally, need to special
1787                            case tex height = 1, since our actual image will have height 2
1788                            and we need to ensure we don't read beyond the texture size
1789                            from user space. */
1790                         if (tex->height == 1) {
1791                                 if (tex_width >= 64 || tex_width <= 16) {
1792                                         RADEON_COPY_MT(buffer, data,
1793                                                 (int)(tex_width * sizeof(u32)));
1794                                 } else if (tex_width == 32) {
1795                                         RADEON_COPY_MT(buffer, data, 16);
1796                                         RADEON_COPY_MT(buffer + 8,
1797                                                        data + 16, 16);
1798                                 }
1799                         } else if (tex_width >= 64 || tex_width == 16) {
1800                                 RADEON_COPY_MT(buffer, data,
1801                                                (int)(dwords * sizeof(u32)));
1802                         } else if (tex_width < 16) {
1803                                 for (i = 0; i < tex->height; i++) {
1804                                         RADEON_COPY_MT(buffer, data, tex_width);
1805                                         buffer += 4;
1806                                         data += tex_width;
1807                                 }
1808                         } else if (tex_width == 32) {
1809                                 /* TODO: make sure this works when not fitting in one buffer
1810                                    (i.e. 32bytes x 2048...) */
1811                                 for (i = 0; i < tex->height; i += 2) {
1812                                         RADEON_COPY_MT(buffer, data, 16);
1813                                         data += 16;
1814                                         RADEON_COPY_MT(buffer + 8, data, 16);
1815                                         data += 16;
1816                                         RADEON_COPY_MT(buffer + 4, data, 16);
1817                                         data += 16;
1818                                         RADEON_COPY_MT(buffer + 12, data, 16);
1819                                         data += 16;
1820                                         buffer += 16;
1821                                 }
1822                         }
1823                 } else {
1824                         if (tex_width >= 32) {
1825                                 /* Texture image width is larger than the minimum, so we
1826                                  * can upload it directly.
1827                                  */
1828                                 RADEON_COPY_MT(buffer, data,
1829                                                (int)(dwords * sizeof(u32)));
1830                         } else {
1831                                 /* Texture image width is less than the minimum, so we
1832                                  * need to pad out each image scanline to the minimum
1833                                  * width.
1834                                  */
1835                                 for (i = 0; i < tex->height; i++) {
1836                                         RADEON_COPY_MT(buffer, data, tex_width);
1837                                         buffer += 8;
1838                                         data += tex_width;
1839                                 }
1840                         }
1841                 }
1842
1843 #undef RADEON_COPY_MT
1844                 buf->file_priv = file_priv;
1845                 buf->used = size;
1846                 offset = dev_priv->gart_buffers_offset + buf->offset;
1847                 BEGIN_RING(9);
1848                 OUT_RING(CP_PACKET3(RADEON_CNTL_BITBLT_MULTI, 5));
1849                 OUT_RING(RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
1850                          RADEON_GMC_DST_PITCH_OFFSET_CNTL |
1851                          RADEON_GMC_BRUSH_NONE |
1852                          (format << 8) |
1853                          RADEON_GMC_SRC_DATATYPE_COLOR |
1854                          RADEON_ROP3_S |
1855                          RADEON_DP_SRC_SOURCE_MEMORY |
1856                          RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS);
1857                 OUT_RING((spitch << 22) | (offset >> 10));
1858                 OUT_RING((texpitch << 22) | (tex->offset >> 10));
1859                 OUT_RING(0);
1860                 OUT_RING((image->x << 16) | image->y);
1861                 OUT_RING((image->width << 16) | height);
1862                 RADEON_WAIT_UNTIL_2D_IDLE();
1863                 ADVANCE_RING();
1864                 COMMIT_RING();
1865
1866                 radeon_cp_discard_buffer(dev, buf);
1867
1868                 /* Update the input parameters for next time */
1869                 image->y += height;
1870                 image->height -= height;
1871                 image->data = (const u8 __user *)image->data + size;
1872         } while (image->height > 0);
1873
1874         /* Flush the pixel cache after the blit completes.  This ensures
1875          * the texture data is written out to memory before rendering
1876          * continues.
1877          */
1878         BEGIN_RING(4);
1879         RADEON_FLUSH_CACHE();
1880         RADEON_WAIT_UNTIL_2D_IDLE();
1881         ADVANCE_RING();
1882         COMMIT_RING();
1883
1884         return 0;
1885 }
1886
1887 static void radeon_cp_dispatch_stipple(struct drm_device * dev, u32 * stipple)
1888 {
1889         drm_radeon_private_t *dev_priv = dev->dev_private;
1890         int i;
1891         RING_LOCALS;
1892         DRM_DEBUG("\n");
1893
1894         BEGIN_RING(35);
1895
1896         OUT_RING(CP_PACKET0(RADEON_RE_STIPPLE_ADDR, 0));
1897         OUT_RING(0x00000000);
1898
1899         OUT_RING(CP_PACKET0_TABLE(RADEON_RE_STIPPLE_DATA, 31));
1900         for (i = 0; i < 32; i++) {
1901                 OUT_RING(stipple[i]);
1902         }
1903
1904         ADVANCE_RING();
1905 }
1906
1907 static void radeon_apply_surface_regs(int surf_index,
1908                                       drm_radeon_private_t *dev_priv)
1909 {
1910         if (!dev_priv->mmio)
1911                 return;
1912
1913         radeon_do_cp_idle(dev_priv);
1914
1915         RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * surf_index,
1916                      dev_priv->surfaces[surf_index].flags);
1917         RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * surf_index,
1918                      dev_priv->surfaces[surf_index].lower);
1919         RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * surf_index,
1920                      dev_priv->surfaces[surf_index].upper);
1921 }
1922
1923 /* Allocates a virtual surface
1924  * doesn't always allocate a real surface, will stretch an existing
1925  * surface when possible.
1926  *
1927  * Note that refcount can be at most 2, since during a free refcount=3
1928  * might mean we have to allocate a new surface which might not always
1929  * be available.
1930  * For example : we allocate three contigous surfaces ABC. If B is
1931  * freed, we suddenly need two surfaces to store A and C, which might
1932  * not always be available.
1933  */
1934 static int alloc_surface(drm_radeon_surface_alloc_t *new,
1935                          drm_radeon_private_t *dev_priv,
1936                          struct drm_file *file_priv)
1937 {
1938         struct radeon_virt_surface *s;
1939         int i;
1940         int virt_surface_index;
1941         uint32_t new_upper, new_lower;
1942
1943         new_lower = new->address;
1944         new_upper = new_lower + new->size - 1;
1945
1946         /* sanity check */
1947         if ((new_lower >= new_upper) || (new->flags == 0) || (new->size == 0) ||
1948             ((new_upper & RADEON_SURF_ADDRESS_FIXED_MASK) !=
1949              RADEON_SURF_ADDRESS_FIXED_MASK)
1950             || ((new_lower & RADEON_SURF_ADDRESS_FIXED_MASK) != 0))
1951                 return -1;
1952
1953         /* make sure there is no overlap with existing surfaces */
1954         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1955                 if ((dev_priv->surfaces[i].refcount != 0) &&
1956                     (((new_lower >= dev_priv->surfaces[i].lower) &&
1957                       (new_lower < dev_priv->surfaces[i].upper)) ||
1958                      ((new_lower < dev_priv->surfaces[i].lower) &&
1959                       (new_upper > dev_priv->surfaces[i].lower)))) {
1960                         return -1;
1961                 }
1962         }
1963
1964         /* find a virtual surface */
1965         for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++)
1966                 if (dev_priv->virt_surfaces[i].file_priv == 0)
1967                         break;
1968         if (i == 2 * RADEON_MAX_SURFACES) {
1969                 return -1;
1970         }
1971         virt_surface_index = i;
1972
1973         /* try to reuse an existing surface */
1974         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1975                 /* extend before */
1976                 if ((dev_priv->surfaces[i].refcount == 1) &&
1977                     (new->flags == dev_priv->surfaces[i].flags) &&
1978                     (new_upper + 1 == dev_priv->surfaces[i].lower)) {
1979                         s = &(dev_priv->virt_surfaces[virt_surface_index]);
1980                         s->surface_index = i;
1981                         s->lower = new_lower;
1982                         s->upper = new_upper;
1983                         s->flags = new->flags;
1984                         s->file_priv = file_priv;
1985                         dev_priv->surfaces[i].refcount++;
1986                         dev_priv->surfaces[i].lower = s->lower;
1987                         radeon_apply_surface_regs(s->surface_index, dev_priv);
1988                         return virt_surface_index;
1989                 }
1990
1991                 /* extend after */
1992                 if ((dev_priv->surfaces[i].refcount == 1) &&
1993                     (new->flags == dev_priv->surfaces[i].flags) &&
1994                     (new_lower == dev_priv->surfaces[i].upper + 1)) {
1995                         s = &(dev_priv->virt_surfaces[virt_surface_index]);
1996                         s->surface_index = i;
1997                         s->lower = new_lower;
1998                         s->upper = new_upper;
1999                         s->flags = new->flags;
2000                         s->file_priv = file_priv;
2001                         dev_priv->surfaces[i].refcount++;
2002                         dev_priv->surfaces[i].upper = s->upper;
2003                         radeon_apply_surface_regs(s->surface_index, dev_priv);
2004                         return virt_surface_index;
2005                 }
2006         }
2007
2008         /* okay, we need a new one */
2009         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
2010                 if (dev_priv->surfaces[i].refcount == 0) {
2011                         s = &(dev_priv->virt_surfaces[virt_surface_index]);
2012                         s->surface_index = i;
2013                         s->lower = new_lower;
2014                         s->upper = new_upper;
2015                         s->flags = new->flags;
2016                         s->file_priv = file_priv;
2017                         dev_priv->surfaces[i].refcount = 1;
2018                         dev_priv->surfaces[i].lower = s->lower;
2019                         dev_priv->surfaces[i].upper = s->upper;
2020                         dev_priv->surfaces[i].flags = s->flags;
2021                         radeon_apply_surface_regs(s->surface_index, dev_priv);
2022                         return virt_surface_index;
2023                 }
2024         }
2025
2026         /* we didn't find anything */
2027         return -1;
2028 }
2029
2030 static int free_surface(struct drm_file *file_priv,
2031                         drm_radeon_private_t * dev_priv,
2032                         int lower)
2033 {
2034         struct radeon_virt_surface *s;
2035         int i;
2036         /* find the virtual surface */
2037         for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
2038                 s = &(dev_priv->virt_surfaces[i]);
2039                 if (s->file_priv) {
2040                         if ((lower == s->lower) && (file_priv == s->file_priv))
2041                         {
2042                                 if (dev_priv->surfaces[s->surface_index].
2043                                     lower == s->lower)
2044                                         dev_priv->surfaces[s->surface_index].
2045                                             lower = s->upper;
2046
2047                                 if (dev_priv->surfaces[s->surface_index].
2048                                     upper == s->upper)
2049                                         dev_priv->surfaces[s->surface_index].
2050                                             upper = s->lower;
2051
2052                                 dev_priv->surfaces[s->surface_index].refcount--;
2053                                 if (dev_priv->surfaces[s->surface_index].
2054                                     refcount == 0)
2055                                         dev_priv->surfaces[s->surface_index].
2056                                             flags = 0;
2057                                 s->file_priv = NULL;
2058                                 radeon_apply_surface_regs(s->surface_index,
2059                                                           dev_priv);
2060                                 return 0;
2061                         }
2062                 }
2063         }
2064         return 1;
2065 }
2066
2067 static void radeon_surfaces_release(struct drm_file *file_priv,
2068                                     drm_radeon_private_t * dev_priv)
2069 {
2070         int i;
2071         for (i = 0; i < 2 * RADEON_MAX_SURFACES; i++) {
2072                 if (dev_priv->virt_surfaces[i].file_priv == file_priv)
2073                         free_surface(file_priv, dev_priv,
2074                                      dev_priv->virt_surfaces[i].lower);
2075         }
2076 }
2077
2078 /* ================================================================
2079  * IOCTL functions
2080  */
2081 static int radeon_surface_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv)
2082 {
2083         drm_radeon_private_t *dev_priv = dev->dev_private;
2084         drm_radeon_surface_alloc_t *alloc = data;
2085
2086         if (alloc_surface(alloc, dev_priv, file_priv) == -1)
2087                 return -EINVAL;
2088         else
2089                 return 0;
2090 }
2091
2092 static int radeon_surface_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
2093 {
2094         drm_radeon_private_t *dev_priv = dev->dev_private;
2095         drm_radeon_surface_free_t *memfree = data;
2096
2097         if (free_surface(file_priv, dev_priv, memfree->address))
2098                 return -EINVAL;
2099         else
2100                 return 0;
2101 }
2102
2103 static int radeon_cp_clear(struct drm_device *dev, void *data, struct drm_file *file_priv)
2104 {
2105         drm_radeon_private_t *dev_priv = dev->dev_private;
2106         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2107         drm_radeon_clear_t *clear = data;
2108         drm_radeon_clear_rect_t depth_boxes[RADEON_NR_SAREA_CLIPRECTS];
2109         DRM_DEBUG("\n");
2110
2111         LOCK_TEST_WITH_RETURN(dev, file_priv);
2112
2113         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2114
2115         if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
2116                 sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
2117
2118         if (DRM_COPY_FROM_USER(&depth_boxes, clear->depth_boxes,
2119                                sarea_priv->nbox * sizeof(depth_boxes[0])))
2120                 return -EFAULT;
2121
2122         radeon_cp_dispatch_clear(dev, clear, depth_boxes);
2123
2124         COMMIT_RING();
2125         return 0;
2126 }
2127
2128 /* Not sure why this isn't set all the time:
2129  */
2130 static int radeon_do_init_pageflip(struct drm_device * dev)
2131 {
2132         drm_radeon_private_t *dev_priv = dev->dev_private;
2133         RING_LOCALS;
2134
2135         DRM_DEBUG("\n");
2136
2137         BEGIN_RING(6);
2138         RADEON_WAIT_UNTIL_3D_IDLE();
2139         OUT_RING(CP_PACKET0(RADEON_CRTC_OFFSET_CNTL, 0));
2140         OUT_RING(RADEON_READ(RADEON_CRTC_OFFSET_CNTL) |
2141                  RADEON_CRTC_OFFSET_FLIP_CNTL);
2142         OUT_RING(CP_PACKET0(RADEON_CRTC2_OFFSET_CNTL, 0));
2143         OUT_RING(RADEON_READ(RADEON_CRTC2_OFFSET_CNTL) |
2144                  RADEON_CRTC_OFFSET_FLIP_CNTL);
2145         ADVANCE_RING();
2146
2147         dev_priv->page_flipping = 1;
2148
2149         if (dev_priv->sarea_priv->pfCurrentPage != 1)
2150                 dev_priv->sarea_priv->pfCurrentPage = 0;
2151
2152         return 0;
2153 }
2154
2155 /* Swapping and flipping are different operations, need different ioctls.
2156  * They can & should be intermixed to support multiple 3d windows.
2157  */
2158 static int radeon_cp_flip(struct drm_device *dev, void *data, struct drm_file *file_priv)
2159 {
2160         drm_radeon_private_t *dev_priv = dev->dev_private;
2161         DRM_DEBUG("\n");
2162
2163         LOCK_TEST_WITH_RETURN(dev, file_priv);
2164
2165         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2166
2167         if (!dev_priv->page_flipping)
2168                 radeon_do_init_pageflip(dev);
2169
2170         radeon_cp_dispatch_flip(dev);
2171
2172         COMMIT_RING();
2173         return 0;
2174 }
2175
2176 static int radeon_cp_swap(struct drm_device *dev, void *data, struct drm_file *file_priv)
2177 {
2178         drm_radeon_private_t *dev_priv = dev->dev_private;
2179         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2180         DRM_DEBUG("\n");
2181
2182         LOCK_TEST_WITH_RETURN(dev, file_priv);
2183
2184         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2185
2186         if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
2187                 sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
2188
2189         radeon_cp_dispatch_swap(dev);
2190         dev_priv->sarea_priv->ctx_owner = 0;
2191
2192         COMMIT_RING();
2193         return 0;
2194 }
2195
2196 static int radeon_cp_vertex(struct drm_device *dev, void *data, struct drm_file *file_priv)
2197 {
2198         drm_radeon_private_t *dev_priv = dev->dev_private;
2199         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2200         struct drm_device_dma *dma = dev->dma;
2201         struct drm_buf *buf;
2202         drm_radeon_vertex_t *vertex = data;
2203         drm_radeon_tcl_prim_t prim;
2204
2205         LOCK_TEST_WITH_RETURN(dev, file_priv);
2206
2207         DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n",
2208                   DRM_CURRENTPID, vertex->idx, vertex->count, vertex->discard);
2209
2210         if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
2211                 DRM_ERROR("buffer index %d (of %d max)\n",
2212                           vertex->idx, dma->buf_count - 1);
2213                 return -EINVAL;
2214         }
2215         if (vertex->prim < 0 || vertex->prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
2216                 DRM_ERROR("buffer prim %d\n", vertex->prim);
2217                 return -EINVAL;
2218         }
2219
2220         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2221         VB_AGE_TEST_WITH_RETURN(dev_priv);
2222
2223         buf = dma->buflist[vertex->idx];
2224
2225         if (buf->file_priv != file_priv) {
2226                 DRM_ERROR("process %d using buffer owned by %p\n",
2227                           DRM_CURRENTPID, buf->file_priv);
2228                 return -EINVAL;
2229         }
2230         if (buf->pending) {
2231                 DRM_ERROR("sending pending buffer %d\n", vertex->idx);
2232                 return -EINVAL;
2233         }
2234
2235         /* Build up a prim_t record:
2236          */
2237         if (vertex->count) {
2238                 buf->used = vertex->count;      /* not used? */
2239
2240                 if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
2241                         if (radeon_emit_state(dev_priv, file_priv,
2242                                               &sarea_priv->context_state,
2243                                               sarea_priv->tex_state,
2244                                               sarea_priv->dirty)) {
2245                                 DRM_ERROR("radeon_emit_state failed\n");
2246                                 return -EINVAL;
2247                         }
2248
2249                         sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
2250                                                RADEON_UPLOAD_TEX1IMAGES |
2251                                                RADEON_UPLOAD_TEX2IMAGES |
2252                                                RADEON_REQUIRE_QUIESCENCE);
2253                 }
2254
2255                 prim.start = 0;
2256                 prim.finish = vertex->count;    /* unused */
2257                 prim.prim = vertex->prim;
2258                 prim.numverts = vertex->count;
2259                 prim.vc_format = dev_priv->sarea_priv->vc_format;
2260
2261                 radeon_cp_dispatch_vertex(dev, buf, &prim);
2262         }
2263
2264         if (vertex->discard) {
2265                 radeon_cp_discard_buffer(dev, buf);
2266         }
2267
2268         COMMIT_RING();
2269         return 0;
2270 }
2271
2272 static int radeon_cp_indices(struct drm_device *dev, void *data, struct drm_file *file_priv)
2273 {
2274         drm_radeon_private_t *dev_priv = dev->dev_private;
2275         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2276         struct drm_device_dma *dma = dev->dma;
2277         struct drm_buf *buf;
2278         drm_radeon_indices_t *elts = data;
2279         drm_radeon_tcl_prim_t prim;
2280         int count;
2281
2282         LOCK_TEST_WITH_RETURN(dev, file_priv);
2283
2284         DRM_DEBUG("pid=%d index=%d start=%d end=%d discard=%d\n",
2285                   DRM_CURRENTPID, elts->idx, elts->start, elts->end,
2286                   elts->discard);
2287
2288         if (elts->idx < 0 || elts->idx >= dma->buf_count) {
2289                 DRM_ERROR("buffer index %d (of %d max)\n",
2290                           elts->idx, dma->buf_count - 1);
2291                 return -EINVAL;
2292         }
2293         if (elts->prim < 0 || elts->prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST) {
2294                 DRM_ERROR("buffer prim %d\n", elts->prim);
2295                 return -EINVAL;
2296         }
2297
2298         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2299         VB_AGE_TEST_WITH_RETURN(dev_priv);
2300
2301         buf = dma->buflist[elts->idx];
2302
2303         if (buf->file_priv != file_priv) {
2304                 DRM_ERROR("process %d using buffer owned by %p\n",
2305                           DRM_CURRENTPID, buf->file_priv);
2306                 return -EINVAL;
2307         }
2308         if (buf->pending) {
2309                 DRM_ERROR("sending pending buffer %d\n", elts->idx);
2310                 return -EINVAL;
2311         }
2312
2313         count = (elts->end - elts->start) / sizeof(u16);
2314         elts->start -= RADEON_INDEX_PRIM_OFFSET;
2315
2316         if (elts->start & 0x7) {
2317                 DRM_ERROR("misaligned buffer 0x%x\n", elts->start);
2318                 return -EINVAL;
2319         }
2320         if (elts->start < buf->used) {
2321                 DRM_ERROR("no header 0x%x - 0x%x\n", elts->start, buf->used);
2322                 return -EINVAL;
2323         }
2324
2325         buf->used = elts->end;
2326
2327         if (sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS) {
2328                 if (radeon_emit_state(dev_priv, file_priv,
2329                                       &sarea_priv->context_state,
2330                                       sarea_priv->tex_state,
2331                                       sarea_priv->dirty)) {
2332                         DRM_ERROR("radeon_emit_state failed\n");
2333                         return -EINVAL;
2334                 }
2335
2336                 sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
2337                                        RADEON_UPLOAD_TEX1IMAGES |
2338                                        RADEON_UPLOAD_TEX2IMAGES |
2339                                        RADEON_REQUIRE_QUIESCENCE);
2340         }
2341
2342         /* Build up a prim_t record:
2343          */
2344         prim.start = elts->start;
2345         prim.finish = elts->end;
2346         prim.prim = elts->prim;
2347         prim.offset = 0;        /* offset from start of dma buffers */
2348         prim.numverts = RADEON_MAX_VB_VERTS;    /* duh */
2349         prim.vc_format = dev_priv->sarea_priv->vc_format;
2350
2351         radeon_cp_dispatch_indices(dev, buf, &prim);
2352         if (elts->discard) {
2353                 radeon_cp_discard_buffer(dev, buf);
2354         }
2355
2356         COMMIT_RING();
2357         return 0;
2358 }
2359
2360 static int radeon_cp_texture(struct drm_device *dev, void *data, struct drm_file *file_priv)
2361 {
2362         drm_radeon_private_t *dev_priv = dev->dev_private;
2363         drm_radeon_texture_t *tex = data;
2364         drm_radeon_tex_image_t image;
2365         int ret;
2366
2367         LOCK_TEST_WITH_RETURN(dev, file_priv);
2368
2369         if (tex->image == NULL) {
2370                 DRM_ERROR("null texture image!\n");
2371                 return -EINVAL;
2372         }
2373
2374         if (DRM_COPY_FROM_USER(&image,
2375                                (drm_radeon_tex_image_t __user *) tex->image,
2376                                sizeof(image)))
2377                 return -EFAULT;
2378
2379         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2380         VB_AGE_TEST_WITH_RETURN(dev_priv);
2381
2382         ret = radeon_cp_dispatch_texture(dev, file_priv, tex, &image);
2383
2384         return ret;
2385 }
2386
2387 static int radeon_cp_stipple(struct drm_device *dev, void *data, struct drm_file *file_priv)
2388 {
2389         drm_radeon_private_t *dev_priv = dev->dev_private;
2390         drm_radeon_stipple_t *stipple = data;
2391         u32 mask[32];
2392
2393         LOCK_TEST_WITH_RETURN(dev, file_priv);
2394
2395         if (DRM_COPY_FROM_USER(&mask, stipple->mask, 32 * sizeof(u32)))
2396                 return -EFAULT;
2397
2398         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2399
2400         radeon_cp_dispatch_stipple(dev, mask);
2401
2402         COMMIT_RING();
2403         return 0;
2404 }
2405
2406 static int radeon_cp_indirect(struct drm_device *dev, void *data, struct drm_file *file_priv)
2407 {
2408         drm_radeon_private_t *dev_priv = dev->dev_private;
2409         struct drm_device_dma *dma = dev->dma;
2410         struct drm_buf *buf;
2411         drm_radeon_indirect_t *indirect = data;
2412         RING_LOCALS;
2413
2414         LOCK_TEST_WITH_RETURN(dev, file_priv);
2415
2416         DRM_DEBUG("indirect: idx=%d s=%d e=%d d=%d\n",
2417                   indirect->idx, indirect->start, indirect->end,
2418                   indirect->discard);
2419
2420         if (indirect->idx < 0 || indirect->idx >= dma->buf_count) {
2421                 DRM_ERROR("buffer index %d (of %d max)\n",
2422                           indirect->idx, dma->buf_count - 1);
2423                 return -EINVAL;
2424         }
2425
2426         buf = dma->buflist[indirect->idx];
2427
2428         if (buf->file_priv != file_priv) {
2429                 DRM_ERROR("process %d using buffer owned by %p\n",
2430                           DRM_CURRENTPID, buf->file_priv);
2431                 return -EINVAL;
2432         }
2433         if (buf->pending) {
2434                 DRM_ERROR("sending pending buffer %d\n", indirect->idx);
2435                 return -EINVAL;
2436         }
2437
2438         if (indirect->start < buf->used) {
2439                 DRM_ERROR("reusing indirect: start=0x%x actual=0x%x\n",
2440                           indirect->start, buf->used);
2441                 return -EINVAL;
2442         }
2443
2444         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2445         VB_AGE_TEST_WITH_RETURN(dev_priv);
2446
2447         buf->used = indirect->end;
2448
2449         /* Wait for the 3D stream to idle before the indirect buffer
2450          * containing 2D acceleration commands is processed.
2451          */
2452         BEGIN_RING(2);
2453
2454         RADEON_WAIT_UNTIL_3D_IDLE();
2455
2456         ADVANCE_RING();
2457
2458         /* Dispatch the indirect buffer full of commands from the
2459          * X server.  This is insecure and is thus only available to
2460          * privileged clients.
2461          */
2462         radeon_cp_dispatch_indirect(dev, buf, indirect->start, indirect->end);
2463         if (indirect->discard) {
2464                 radeon_cp_discard_buffer(dev, buf);
2465         }
2466
2467         COMMIT_RING();
2468         return 0;
2469 }
2470
2471 static int radeon_cp_vertex2(struct drm_device *dev, void *data, struct drm_file *file_priv)
2472 {
2473         drm_radeon_private_t *dev_priv = dev->dev_private;
2474         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
2475         struct drm_device_dma *dma = dev->dma;
2476         struct drm_buf *buf;
2477         drm_radeon_vertex2_t *vertex = data;
2478         int i;
2479         unsigned char laststate;
2480
2481         LOCK_TEST_WITH_RETURN(dev, file_priv);
2482
2483         DRM_DEBUG("pid=%d index=%d discard=%d\n",
2484                   DRM_CURRENTPID, vertex->idx, vertex->discard);
2485
2486         if (vertex->idx < 0 || vertex->idx >= dma->buf_count) {
2487                 DRM_ERROR("buffer index %d (of %d max)\n",
2488                           vertex->idx, dma->buf_count - 1);
2489                 return -EINVAL;
2490         }
2491
2492         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2493         VB_AGE_TEST_WITH_RETURN(dev_priv);
2494
2495         buf = dma->buflist[vertex->idx];
2496
2497         if (buf->file_priv != file_priv) {
2498                 DRM_ERROR("process %d using buffer owned by %p\n",
2499                           DRM_CURRENTPID, buf->file_priv);
2500                 return -EINVAL;
2501         }
2502
2503         if (buf->pending) {
2504                 DRM_ERROR("sending pending buffer %d\n", vertex->idx);
2505                 return -EINVAL;
2506         }
2507
2508         if (sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS)
2509                 return -EINVAL;
2510
2511         for (laststate = 0xff, i = 0; i < vertex->nr_prims; i++) {
2512                 drm_radeon_prim_t prim;
2513                 drm_radeon_tcl_prim_t tclprim;
2514
2515                 if (DRM_COPY_FROM_USER(&prim, &vertex->prim[i], sizeof(prim)))
2516                         return -EFAULT;
2517
2518                 if (prim.stateidx != laststate) {
2519                         drm_radeon_state_t state;
2520
2521                         if (DRM_COPY_FROM_USER(&state,
2522                                                &vertex->state[prim.stateidx],
2523                                                sizeof(state)))
2524                                 return -EFAULT;
2525
2526                         if (radeon_emit_state2(dev_priv, file_priv, &state)) {
2527                                 DRM_ERROR("radeon_emit_state2 failed\n");
2528                                 return -EINVAL;
2529                         }
2530
2531                         laststate = prim.stateidx;
2532                 }
2533
2534                 tclprim.start = prim.start;
2535                 tclprim.finish = prim.finish;
2536                 tclprim.prim = prim.prim;
2537                 tclprim.vc_format = prim.vc_format;
2538
2539                 if (prim.prim & RADEON_PRIM_WALK_IND) {
2540                         tclprim.offset = prim.numverts * 64;
2541                         tclprim.numverts = RADEON_MAX_VB_VERTS; /* duh */
2542
2543                         radeon_cp_dispatch_indices(dev, buf, &tclprim);
2544                 } else {
2545                         tclprim.numverts = prim.numverts;
2546                         tclprim.offset = 0;     /* not used */
2547
2548                         radeon_cp_dispatch_vertex(dev, buf, &tclprim);
2549                 }
2550
2551                 if (sarea_priv->nbox == 1)
2552                         sarea_priv->nbox = 0;
2553         }
2554
2555         if (vertex->discard) {
2556                 radeon_cp_discard_buffer(dev, buf);
2557         }
2558
2559         COMMIT_RING();
2560         return 0;
2561 }
2562
2563 static int radeon_emit_packets(drm_radeon_private_t * dev_priv,
2564                                struct drm_file *file_priv,
2565                                drm_radeon_cmd_header_t header,
2566                                drm_radeon_kcmd_buffer_t *cmdbuf)
2567 {
2568         int id = (int)header.packet.packet_id;
2569         int sz, reg;
2570         int *data = (int *)cmdbuf->buf;
2571         RING_LOCALS;
2572
2573         if (id >= RADEON_MAX_STATE_PACKETS)
2574                 return -EINVAL;
2575
2576         sz = packet[id].len;
2577         reg = packet[id].start;
2578
2579         if (sz * sizeof(int) > cmdbuf->bufsz) {
2580                 DRM_ERROR("Packet size provided larger than data provided\n");
2581                 return -EINVAL;
2582         }
2583
2584         if (radeon_check_and_fixup_packets(dev_priv, file_priv, id, data)) {
2585                 DRM_ERROR("Packet verification failed\n");
2586                 return -EINVAL;
2587         }
2588
2589         BEGIN_RING(sz + 1);
2590         OUT_RING(CP_PACKET0(reg, (sz - 1)));
2591         OUT_RING_TABLE(data, sz);
2592         ADVANCE_RING();
2593
2594         cmdbuf->buf += sz * sizeof(int);
2595         cmdbuf->bufsz -= sz * sizeof(int);
2596         return 0;
2597 }
2598
2599 static __inline__ int radeon_emit_scalars(drm_radeon_private_t *dev_priv,
2600                                           drm_radeon_cmd_header_t header,
2601                                           drm_radeon_kcmd_buffer_t *cmdbuf)
2602 {
2603         int sz = header.scalars.count;
2604         int start = header.scalars.offset;
2605         int stride = header.scalars.stride;
2606         RING_LOCALS;
2607
2608         BEGIN_RING(3 + sz);
2609         OUT_RING(CP_PACKET0(RADEON_SE_TCL_SCALAR_INDX_REG, 0));
2610         OUT_RING(start | (stride << RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT));
2611         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_SCALAR_DATA_REG, sz - 1));
2612         OUT_RING_TABLE(cmdbuf->buf, sz);
2613         ADVANCE_RING();
2614         cmdbuf->buf += sz * sizeof(int);
2615         cmdbuf->bufsz -= sz * sizeof(int);
2616         return 0;
2617 }
2618
2619 /* God this is ugly
2620  */
2621 static __inline__ int radeon_emit_scalars2(drm_radeon_private_t *dev_priv,
2622                                            drm_radeon_cmd_header_t header,
2623                                            drm_radeon_kcmd_buffer_t *cmdbuf)
2624 {
2625         int sz = header.scalars.count;
2626         int start = ((unsigned int)header.scalars.offset) + 0x100;
2627         int stride = header.scalars.stride;
2628         RING_LOCALS;
2629
2630         BEGIN_RING(3 + sz);
2631         OUT_RING(CP_PACKET0(RADEON_SE_TCL_SCALAR_INDX_REG, 0));
2632         OUT_RING(start | (stride << RADEON_SCAL_INDX_DWORD_STRIDE_SHIFT));
2633         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_SCALAR_DATA_REG, sz - 1));
2634         OUT_RING_TABLE(cmdbuf->buf, sz);
2635         ADVANCE_RING();
2636         cmdbuf->buf += sz * sizeof(int);
2637         cmdbuf->bufsz -= sz * sizeof(int);
2638         return 0;
2639 }
2640
2641 static __inline__ int radeon_emit_vectors(drm_radeon_private_t *dev_priv,
2642                                           drm_radeon_cmd_header_t header,
2643                                           drm_radeon_kcmd_buffer_t *cmdbuf)
2644 {
2645         int sz = header.vectors.count;
2646         int start = header.vectors.offset;
2647         int stride = header.vectors.stride;
2648         RING_LOCALS;
2649
2650         BEGIN_RING(5 + sz);
2651         OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
2652         OUT_RING(CP_PACKET0(RADEON_SE_TCL_VECTOR_INDX_REG, 0));
2653         OUT_RING(start | (stride << RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT));
2654         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_VECTOR_DATA_REG, (sz - 1)));
2655         OUT_RING_TABLE(cmdbuf->buf, sz);
2656         ADVANCE_RING();
2657
2658         cmdbuf->buf += sz * sizeof(int);
2659         cmdbuf->bufsz -= sz * sizeof(int);
2660         return 0;
2661 }
2662
2663 static __inline__ int radeon_emit_veclinear(drm_radeon_private_t *dev_priv,
2664                                           drm_radeon_cmd_header_t header,
2665                                           drm_radeon_kcmd_buffer_t *cmdbuf)
2666 {
2667         int sz = header.veclinear.count * 4;
2668         int start = header.veclinear.addr_lo | (header.veclinear.addr_hi << 8);
2669         RING_LOCALS;
2670
2671         if (!sz)
2672                 return 0;
2673         if (sz * 4 > cmdbuf->bufsz)
2674                 return -EINVAL;
2675
2676         BEGIN_RING(5 + sz);
2677         OUT_RING_REG(RADEON_SE_TCL_STATE_FLUSH, 0);
2678         OUT_RING(CP_PACKET0(RADEON_SE_TCL_VECTOR_INDX_REG, 0));
2679         OUT_RING(start | (1 << RADEON_VEC_INDX_OCTWORD_STRIDE_SHIFT));
2680         OUT_RING(CP_PACKET0_TABLE(RADEON_SE_TCL_VECTOR_DATA_REG, (sz - 1)));
2681         OUT_RING_TABLE(cmdbuf->buf, sz);
2682         ADVANCE_RING();
2683
2684         cmdbuf->buf += sz * sizeof(int);
2685         cmdbuf->bufsz -= sz * sizeof(int);
2686         return 0;
2687 }
2688
2689 static int radeon_emit_packet3(struct drm_device * dev,
2690                                struct drm_file *file_priv,
2691                                drm_radeon_kcmd_buffer_t *cmdbuf)
2692 {
2693         drm_radeon_private_t *dev_priv = dev->dev_private;
2694         unsigned int cmdsz;
2695         int ret;
2696         RING_LOCALS;
2697
2698         DRM_DEBUG("\n");
2699
2700         if ((ret = radeon_check_and_fixup_packet3(dev_priv, file_priv,
2701                                                   cmdbuf, &cmdsz))) {
2702                 DRM_ERROR("Packet verification failed\n");
2703                 return ret;
2704         }
2705
2706         BEGIN_RING(cmdsz);
2707         OUT_RING_TABLE(cmdbuf->buf, cmdsz);
2708         ADVANCE_RING();
2709
2710         cmdbuf->buf += cmdsz * 4;
2711         cmdbuf->bufsz -= cmdsz * 4;
2712         return 0;
2713 }
2714
2715 static int radeon_emit_packet3_cliprect(struct drm_device *dev,
2716                                         struct drm_file *file_priv,
2717                                         drm_radeon_kcmd_buffer_t *cmdbuf,
2718                                         int orig_nbox)
2719 {
2720         drm_radeon_private_t *dev_priv = dev->dev_private;
2721         struct drm_clip_rect box;
2722         unsigned int cmdsz;
2723         int ret;
2724         struct drm_clip_rect __user *boxes = cmdbuf->boxes;
2725         int i = 0;
2726         RING_LOCALS;
2727
2728         DRM_DEBUG("\n");
2729
2730         if ((ret = radeon_check_and_fixup_packet3(dev_priv, file_priv,
2731                                                   cmdbuf, &cmdsz))) {
2732                 DRM_ERROR("Packet verification failed\n");
2733                 return ret;
2734         }
2735
2736         if (!orig_nbox)
2737                 goto out;
2738
2739         do {
2740                 if (i < cmdbuf->nbox) {
2741                         if (DRM_COPY_FROM_USER(&box, &boxes[i], sizeof(box)))
2742                                 return -EFAULT;
2743                         /* FIXME The second and subsequent times round
2744                          * this loop, send a WAIT_UNTIL_3D_IDLE before
2745                          * calling emit_clip_rect(). This fixes a
2746                          * lockup on fast machines when sending
2747                          * several cliprects with a cmdbuf, as when
2748                          * waving a 2D window over a 3D
2749                          * window. Something in the commands from user
2750                          * space seems to hang the card when they're
2751                          * sent several times in a row. That would be
2752                          * the correct place to fix it but this works
2753                          * around it until I can figure that out - Tim
2754                          * Smith */
2755                         if (i) {
2756                                 BEGIN_RING(2);
2757                                 RADEON_WAIT_UNTIL_3D_IDLE();
2758                                 ADVANCE_RING();
2759                         }
2760                         radeon_emit_clip_rect(dev_priv, &box);
2761                 }
2762
2763                 BEGIN_RING(cmdsz);
2764                 OUT_RING_TABLE(cmdbuf->buf, cmdsz);
2765                 ADVANCE_RING();
2766
2767         } while (++i < cmdbuf->nbox);
2768         if (cmdbuf->nbox == 1)
2769                 cmdbuf->nbox = 0;
2770
2771       out:
2772         cmdbuf->buf += cmdsz * 4;
2773         cmdbuf->bufsz -= cmdsz * 4;
2774         return 0;
2775 }
2776
2777 static int radeon_emit_wait(struct drm_device * dev, int flags)
2778 {
2779         drm_radeon_private_t *dev_priv = dev->dev_private;
2780         RING_LOCALS;
2781
2782         DRM_DEBUG("%s: %x\n", __FUNCTION__, flags);
2783         switch (flags) {
2784         case RADEON_WAIT_2D:
2785                 BEGIN_RING(2);
2786                 RADEON_WAIT_UNTIL_2D_IDLE();
2787                 ADVANCE_RING();
2788                 break;
2789         case RADEON_WAIT_3D:
2790                 BEGIN_RING(2);
2791                 RADEON_WAIT_UNTIL_3D_IDLE();
2792                 ADVANCE_RING();
2793                 break;
2794         case RADEON_WAIT_2D | RADEON_WAIT_3D:
2795                 BEGIN_RING(2);
2796                 RADEON_WAIT_UNTIL_IDLE();
2797                 ADVANCE_RING();
2798                 break;
2799         default:
2800                 return -EINVAL;
2801         }
2802
2803         return 0;
2804 }
2805
2806 static int radeon_cp_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_priv)
2807 {
2808         drm_radeon_private_t *dev_priv = dev->dev_private;
2809         struct drm_device_dma *dma = dev->dma;
2810         struct drm_buf *buf = NULL;
2811         int idx;
2812         drm_radeon_kcmd_buffer_t *cmdbuf = data;
2813         drm_radeon_cmd_header_t header;
2814         int orig_nbox, orig_bufsz;
2815         char *kbuf = NULL;
2816
2817         LOCK_TEST_WITH_RETURN(dev, file_priv);
2818
2819         RING_SPACE_TEST_WITH_RETURN(dev_priv);
2820         VB_AGE_TEST_WITH_RETURN(dev_priv);
2821
2822         if (cmdbuf->bufsz > 64 * 1024 || cmdbuf->bufsz < 0) {
2823                 return -EINVAL;
2824         }
2825
2826         /* Allocate an in-kernel area and copy in the cmdbuf.  Do this to avoid
2827          * races between checking values and using those values in other code,
2828          * and simply to avoid a lot of function calls to copy in data.
2829          */
2830         orig_bufsz = cmdbuf->bufsz;
2831         if (orig_bufsz != 0) {
2832                 kbuf = drm_alloc(cmdbuf->bufsz, DRM_MEM_DRIVER);
2833                 if (kbuf == NULL)
2834                         return -ENOMEM;
2835                 if (DRM_COPY_FROM_USER(kbuf, (void __user *)cmdbuf->buf,
2836                                        cmdbuf->bufsz)) {
2837                         drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
2838                         return -EFAULT;
2839                 }
2840                 cmdbuf->buf = kbuf;
2841         }
2842
2843         orig_nbox = cmdbuf->nbox;
2844
2845         if (dev_priv->microcode_version == UCODE_R300) {
2846                 int temp;
2847                 temp = r300_do_cp_cmdbuf(dev, file_priv, cmdbuf);
2848
2849                 if (orig_bufsz != 0)
2850                         drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
2851
2852                 return temp;
2853         }
2854
2855         /* microcode_version != r300 */
2856         while (cmdbuf->bufsz >= sizeof(header)) {
2857
2858                 header.i = *(int *)cmdbuf->buf;
2859                 cmdbuf->buf += sizeof(header);
2860                 cmdbuf->bufsz -= sizeof(header);
2861
2862                 switch (header.header.cmd_type) {
2863                 case RADEON_CMD_PACKET:
2864                         DRM_DEBUG("RADEON_CMD_PACKET\n");
2865                         if (radeon_emit_packets
2866                             (dev_priv, file_priv, header, cmdbuf)) {
2867                                 DRM_ERROR("radeon_emit_packets failed\n");
2868                                 goto err;
2869                         }
2870                         break;
2871
2872                 case RADEON_CMD_SCALARS:
2873                         DRM_DEBUG("RADEON_CMD_SCALARS\n");
2874                         if (radeon_emit_scalars(dev_priv, header, cmdbuf)) {
2875                                 DRM_ERROR("radeon_emit_scalars failed\n");
2876                                 goto err;
2877                         }
2878                         break;
2879
2880                 case RADEON_CMD_VECTORS:
2881                         DRM_DEBUG("RADEON_CMD_VECTORS\n");
2882                         if (radeon_emit_vectors(dev_priv, header, cmdbuf)) {
2883                                 DRM_ERROR("radeon_emit_vectors failed\n");
2884                                 goto err;
2885                         }
2886                         break;
2887
2888                 case RADEON_CMD_DMA_DISCARD:
2889                         DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n");
2890                         idx = header.dma.buf_idx;
2891                         if (idx < 0 || idx >= dma->buf_count) {
2892                                 DRM_ERROR("buffer index %d (of %d max)\n",
2893                                           idx, dma->buf_count - 1);
2894                                 goto err;
2895                         }
2896
2897                         buf = dma->buflist[idx];
2898                         if (buf->file_priv != file_priv || buf->pending) {
2899                                 DRM_ERROR("bad buffer %p %p %d\n",
2900                                           buf->file_priv, file_priv,
2901                                           buf->pending);
2902                                 goto err;
2903                         }
2904
2905                         radeon_cp_discard_buffer(dev, buf);
2906                         break;
2907
2908                 case RADEON_CMD_PACKET3:
2909                         DRM_DEBUG("RADEON_CMD_PACKET3\n");
2910                         if (radeon_emit_packet3(dev, file_priv, cmdbuf)) {
2911                                 DRM_ERROR("radeon_emit_packet3 failed\n");
2912                                 goto err;
2913                         }
2914                         break;
2915
2916                 case RADEON_CMD_PACKET3_CLIP:
2917                         DRM_DEBUG("RADEON_CMD_PACKET3_CLIP\n");
2918                         if (radeon_emit_packet3_cliprect
2919                             (dev, file_priv, cmdbuf, orig_nbox)) {
2920                                 DRM_ERROR("radeon_emit_packet3_clip failed\n");
2921                                 goto err;
2922                         }
2923                         break;
2924
2925                 case RADEON_CMD_SCALARS2:
2926                         DRM_DEBUG("RADEON_CMD_SCALARS2\n");
2927                         if (radeon_emit_scalars2(dev_priv, header, cmdbuf)) {
2928                                 DRM_ERROR("radeon_emit_scalars2 failed\n");
2929                                 goto err;
2930                         }
2931                         break;
2932
2933                 case RADEON_CMD_WAIT:
2934                         DRM_DEBUG("RADEON_CMD_WAIT\n");
2935                         if (radeon_emit_wait(dev, header.wait.flags)) {
2936                                 DRM_ERROR("radeon_emit_wait failed\n");
2937                                 goto err;
2938                         }
2939                         break;
2940                 case RADEON_CMD_VECLINEAR:
2941                         DRM_DEBUG("RADEON_CMD_VECLINEAR\n");
2942                         if (radeon_emit_veclinear(dev_priv, header, cmdbuf)) {
2943                                 DRM_ERROR("radeon_emit_veclinear failed\n");
2944                                 goto err;
2945                         }
2946                         break;
2947
2948                 default:
2949                         DRM_ERROR("bad cmd_type %d at %p\n",
2950                                   header.header.cmd_type,
2951                                   cmdbuf->buf - sizeof(header));
2952                         goto err;
2953                 }
2954         }
2955
2956         if (orig_bufsz != 0)
2957                 drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
2958
2959         DRM_DEBUG("DONE\n");
2960         COMMIT_RING();
2961         return 0;
2962
2963       err:
2964         if (orig_bufsz != 0)
2965                 drm_free(kbuf, orig_bufsz, DRM_MEM_DRIVER);
2966         return -EINVAL;
2967 }
2968
2969 static int radeon_cp_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
2970 {
2971         drm_radeon_private_t *dev_priv = dev->dev_private;
2972         drm_radeon_getparam_t *param = data;
2973         int value;
2974
2975         DRM_DEBUG("pid=%d\n", DRM_CURRENTPID);
2976
2977         switch (param->param) {
2978         case RADEON_PARAM_GART_BUFFER_OFFSET:
2979                 value = dev_priv->gart_buffers_offset;
2980                 break;
2981         case RADEON_PARAM_LAST_FRAME:
2982                 dev_priv->stats.last_frame_reads++;
2983                 value = GET_SCRATCH(0);
2984                 break;
2985         case RADEON_PARAM_LAST_DISPATCH:
2986                 value = GET_SCRATCH(1);
2987                 break;
2988         case RADEON_PARAM_LAST_CLEAR:
2989                 dev_priv->stats.last_clear_reads++;
2990                 value = GET_SCRATCH(2);
2991                 break;
2992         case RADEON_PARAM_IRQ_NR:
2993                 value = dev->irq;
2994                 break;
2995         case RADEON_PARAM_GART_BASE:
2996                 value = dev_priv->gart_vm_start;
2997                 break;
2998         case RADEON_PARAM_REGISTER_HANDLE:
2999                 value = dev_priv->mmio->offset;
3000                 break;
3001         case RADEON_PARAM_STATUS_HANDLE:
3002                 value = dev_priv->ring_rptr_offset;
3003                 break;
3004 #if BITS_PER_LONG == 32
3005                 /*
3006                  * This ioctl() doesn't work on 64-bit platforms because hw_lock is a
3007                  * pointer which can't fit into an int-sized variable.  According to
3008                  * Michel Dänzer, the ioctl() is only used on embedded platforms, so
3009                  * not supporting it shouldn't be a problem.  If the same functionality
3010                  * is needed on 64-bit platforms, a new ioctl() would have to be added,
3011                  * so backwards-compatibility for the embedded platforms can be
3012                  * maintained.  --davidm 4-Feb-2004.
3013                  */
3014         case RADEON_PARAM_SAREA_HANDLE:
3015                 /* The lock is the first dword in the sarea. */
3016                 value = (long)dev->lock.hw_lock;
3017                 break;
3018 #endif
3019         case RADEON_PARAM_GART_TEX_HANDLE:
3020                 value = dev_priv->gart_textures_offset;
3021                 break;
3022         case RADEON_PARAM_SCRATCH_OFFSET:
3023                 if (!dev_priv->writeback_works)
3024                         return -EINVAL;
3025                 value = RADEON_SCRATCH_REG_OFFSET;
3026                 break;
3027         case RADEON_PARAM_CARD_TYPE:
3028                 if (dev_priv->flags & RADEON_IS_PCIE)
3029                         value = RADEON_CARD_PCIE;
3030                 else if (dev_priv->flags & RADEON_IS_AGP)
3031                         value = RADEON_CARD_AGP;
3032                 else
3033                         value = RADEON_CARD_PCI;
3034                 break;
3035         case RADEON_PARAM_VBLANK_CRTC:
3036                 value = radeon_vblank_crtc_get(dev);
3037                 break;
3038         default:
3039                 DRM_DEBUG("Invalid parameter %d\n", param->param);
3040                 return -EINVAL;
3041         }
3042
3043         if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
3044                 DRM_ERROR("copy_to_user\n");
3045                 return -EFAULT;
3046         }
3047
3048         return 0;
3049 }
3050
3051 static int radeon_cp_setparam(struct drm_device *dev, void *data, struct drm_file *file_priv)
3052 {
3053         drm_radeon_private_t *dev_priv = dev->dev_private;
3054         drm_radeon_setparam_t *sp = data;
3055         struct drm_radeon_driver_file_fields *radeon_priv;
3056
3057         switch (sp->param) {
3058         case RADEON_SETPARAM_FB_LOCATION:
3059                 radeon_priv = file_priv->driver_priv;
3060                 radeon_priv->radeon_fb_delta = dev_priv->fb_location -
3061                     sp->value;
3062                 break;
3063         case RADEON_SETPARAM_SWITCH_TILING:
3064                 if (sp->value == 0) {
3065                         DRM_DEBUG("color tiling disabled\n");
3066                         dev_priv->front_pitch_offset &= ~RADEON_DST_TILE_MACRO;
3067                         dev_priv->back_pitch_offset &= ~RADEON_DST_TILE_MACRO;
3068                         dev_priv->sarea_priv->tiling_enabled = 0;
3069                 } else if (sp->value == 1) {
3070                         DRM_DEBUG("color tiling enabled\n");
3071                         dev_priv->front_pitch_offset |= RADEON_DST_TILE_MACRO;
3072                         dev_priv->back_pitch_offset |= RADEON_DST_TILE_MACRO;
3073                         dev_priv->sarea_priv->tiling_enabled = 1;
3074                 }
3075                 break;
3076         case RADEON_SETPARAM_PCIGART_LOCATION:
3077                 dev_priv->pcigart_offset = sp->value;
3078                 dev_priv->pcigart_offset_set = 1;
3079                 break;
3080         case RADEON_SETPARAM_NEW_MEMMAP:
3081                 dev_priv->new_memmap = sp->value;
3082                 break;
3083         case RADEON_SETPARAM_PCIGART_TABLE_SIZE:
3084                 dev_priv->gart_info.table_size = sp->value;
3085                 if (dev_priv->gart_info.table_size < RADEON_PCIGART_TABLE_SIZE)
3086                         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
3087                 break;
3088         case RADEON_SETPARAM_VBLANK_CRTC:
3089                 return radeon_vblank_crtc_set(dev, sp->value);
3090                 break;
3091         default:
3092                 DRM_DEBUG("Invalid parameter %d\n", sp->param);
3093                 return -EINVAL;
3094         }
3095
3096         return 0;
3097 }
3098
3099 /* When a client dies:
3100  *    - Check for and clean up flipped page state
3101  *    - Free any alloced GART memory.
3102  *    - Free any alloced radeon surfaces.
3103  *
3104  * DRM infrastructure takes care of reclaiming dma buffers.
3105  */
3106 void radeon_driver_preclose(struct drm_device *dev, struct drm_file *file_priv)
3107 {
3108         if (dev->dev_private) {
3109                 drm_radeon_private_t *dev_priv = dev->dev_private;
3110                 dev_priv->page_flipping = 0;
3111                 radeon_mem_release(file_priv, dev_priv->gart_heap);
3112                 radeon_mem_release(file_priv, dev_priv->fb_heap);
3113                 radeon_surfaces_release(file_priv, dev_priv);
3114         }
3115 }
3116
3117 void radeon_driver_lastclose(struct drm_device *dev)
3118 {
3119         if (dev->dev_private) {
3120                 drm_radeon_private_t *dev_priv = dev->dev_private;
3121
3122                 if (dev_priv->sarea_priv &&
3123                     dev_priv->sarea_priv->pfCurrentPage != 0)
3124                         radeon_cp_dispatch_flip(dev);
3125         }
3126
3127         radeon_do_release(dev);
3128 }
3129
3130 int radeon_driver_open(struct drm_device *dev, struct drm_file *file_priv)
3131 {
3132         drm_radeon_private_t *dev_priv = dev->dev_private;
3133         struct drm_radeon_driver_file_fields *radeon_priv;
3134
3135         DRM_DEBUG("\n");
3136         radeon_priv =
3137             (struct drm_radeon_driver_file_fields *)
3138             drm_alloc(sizeof(*radeon_priv), DRM_MEM_FILES);
3139
3140         if (!radeon_priv)
3141                 return -ENOMEM;
3142
3143         file_priv->driver_priv = radeon_priv;
3144
3145         if (dev_priv)
3146                 radeon_priv->radeon_fb_delta = dev_priv->fb_location;
3147         else
3148                 radeon_priv->radeon_fb_delta = 0;
3149         return 0;
3150 }
3151
3152 void radeon_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
3153 {
3154         struct drm_radeon_driver_file_fields *radeon_priv =
3155             file_priv->driver_priv;
3156
3157         drm_free(radeon_priv, sizeof(*radeon_priv), DRM_MEM_FILES);
3158 }
3159
3160 struct drm_ioctl_desc radeon_ioctls[] = {
3161         DRM_IOCTL_DEF(DRM_RADEON_CP_INIT, radeon_cp_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
3162         DRM_IOCTL_DEF(DRM_RADEON_CP_START, radeon_cp_start, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
3163         DRM_IOCTL_DEF(DRM_RADEON_CP_STOP, radeon_cp_stop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
3164         DRM_IOCTL_DEF(DRM_RADEON_CP_RESET, radeon_cp_reset, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
3165         DRM_IOCTL_DEF(DRM_RADEON_CP_IDLE, radeon_cp_idle, DRM_AUTH),
3166         DRM_IOCTL_DEF(DRM_RADEON_CP_RESUME, radeon_cp_resume, DRM_AUTH),
3167         DRM_IOCTL_DEF(DRM_RADEON_RESET, radeon_engine_reset, DRM_AUTH),
3168         DRM_IOCTL_DEF(DRM_RADEON_FULLSCREEN, radeon_fullscreen, DRM_AUTH),
3169         DRM_IOCTL_DEF(DRM_RADEON_SWAP, radeon_cp_swap, DRM_AUTH),
3170         DRM_IOCTL_DEF(DRM_RADEON_CLEAR, radeon_cp_clear, DRM_AUTH),
3171         DRM_IOCTL_DEF(DRM_RADEON_VERTEX, radeon_cp_vertex, DRM_AUTH),
3172         DRM_IOCTL_DEF(DRM_RADEON_INDICES, radeon_cp_indices, DRM_AUTH),
3173         DRM_IOCTL_DEF(DRM_RADEON_TEXTURE, radeon_cp_texture, DRM_AUTH),
3174         DRM_IOCTL_DEF(DRM_RADEON_STIPPLE, radeon_cp_stipple, DRM_AUTH),
3175         DRM_IOCTL_DEF(DRM_RADEON_INDIRECT, radeon_cp_indirect, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
3176         DRM_IOCTL_DEF(DRM_RADEON_VERTEX2, radeon_cp_vertex2, DRM_AUTH),
3177         DRM_IOCTL_DEF(DRM_RADEON_CMDBUF, radeon_cp_cmdbuf, DRM_AUTH),
3178         DRM_IOCTL_DEF(DRM_RADEON_GETPARAM, radeon_cp_getparam, DRM_AUTH),
3179         DRM_IOCTL_DEF(DRM_RADEON_FLIP, radeon_cp_flip, DRM_AUTH),
3180         DRM_IOCTL_DEF(DRM_RADEON_ALLOC, radeon_mem_alloc, DRM_AUTH),
3181         DRM_IOCTL_DEF(DRM_RADEON_FREE, radeon_mem_free, DRM_AUTH),
3182         DRM_IOCTL_DEF(DRM_RADEON_INIT_HEAP, radeon_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
3183         DRM_IOCTL_DEF(DRM_RADEON_IRQ_EMIT, radeon_irq_emit, DRM_AUTH),
3184         DRM_IOCTL_DEF(DRM_RADEON_IRQ_WAIT, radeon_irq_wait, DRM_AUTH),
3185         DRM_IOCTL_DEF(DRM_RADEON_SETPARAM, radeon_cp_setparam, DRM_AUTH),
3186         DRM_IOCTL_DEF(DRM_RADEON_SURF_ALLOC, radeon_surface_alloc, DRM_AUTH),
3187         DRM_IOCTL_DEF(DRM_RADEON_SURF_FREE, radeon_surface_free, DRM_AUTH)
3188 };
3189
3190 int radeon_max_ioctl = DRM_ARRAY_SIZE(radeon_ioctls);