2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/pci.h>
37 #include <linux/timer.h>
38 #include "firmware_exports.h"
42 #define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
43 #define Q_FULL(rptr,wptr,size_log2) ( (((wptr)-(rptr))>>(size_log2)) && \
45 #define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
46 #define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
47 #define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
48 #define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
50 static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)
52 writel(((1<<31) | qpid), doorbell);
55 #define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))
58 T3_COMPLETION_FLAG = 0x01,
59 T3_NOTIFY_FLAG = 0x02,
60 T3_SOLICITED_EVENT_FLAG = 0x04,
61 T3_READ_FENCE_FLAG = 0x08,
62 T3_LOCAL_FENCE_FLAG = 0x10
63 } __attribute__ ((packed));
66 T3_WR_BP = FW_WROPCODE_RI_BYPASS,
67 T3_WR_SEND = FW_WROPCODE_RI_SEND,
68 T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
69 T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
70 T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
71 T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
72 T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
73 T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
74 T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
75 } __attribute__ ((packed));
78 T3_RDMA_WRITE, /* IETF RDMAP v1.0 ... */
86 T3_RDMA_INIT, /* CHELSIO RI specific ... */
92 } __attribute__ ((packed));
94 static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
97 case T3_WR_BP: return T3_BYPASS;
98 case T3_WR_SEND: return T3_SEND;
99 case T3_WR_WRITE: return T3_RDMA_WRITE;
100 case T3_WR_READ: return T3_READ_REQ;
101 case T3_WR_INV_STAG: return T3_LOCAL_INV;
102 case T3_WR_BIND: return T3_BIND_MW;
103 case T3_WR_INIT: return T3_RDMA_INIT;
104 case T3_WR_QP_MOD: return T3_QP_MOD;
111 /* Work request id */
120 #define WRID(wrid) (wrid.id1)
121 #define WRID_GEN(wrid) (wrid.id0.wr_gen)
122 #define WRID_IDX(wrid) (wrid.id0.wr_idx)
123 #define WRID_LO(wrid) (wrid.id0.wr_lo)
126 __be32 op_seop_flags;
130 #define S_FW_RIWR_OP 24
131 #define M_FW_RIWR_OP 0xff
132 #define V_FW_RIWR_OP(x) ((x) << S_FW_RIWR_OP)
133 #define G_FW_RIWR_OP(x) ((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)
135 #define S_FW_RIWR_SOPEOP 22
136 #define M_FW_RIWR_SOPEOP 0x3
137 #define V_FW_RIWR_SOPEOP(x) ((x) << S_FW_RIWR_SOPEOP)
139 #define S_FW_RIWR_FLAGS 8
140 #define M_FW_RIWR_FLAGS 0x3fffff
141 #define V_FW_RIWR_FLAGS(x) ((x) << S_FW_RIWR_FLAGS)
142 #define G_FW_RIWR_FLAGS(x) ((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)
144 #define S_FW_RIWR_TID 8
145 #define V_FW_RIWR_TID(x) ((x) << S_FW_RIWR_TID)
147 #define S_FW_RIWR_LEN 0
148 #define V_FW_RIWR_LEN(x) ((x) << S_FW_RIWR_LEN)
150 #define S_FW_RIWR_GEN 31
151 #define V_FW_RIWR_GEN(x) ((x) << S_FW_RIWR_GEN)
159 /* If num_sgle is zero, flit 5+ contains immediate data.*/
161 struct fw_riwrh wrh; /* 0 */
162 union t3_wrid wrid; /* 1 */
169 struct t3_sge sgl[T3_MAX_SGE]; /* 4+ */
172 struct t3_local_inv_wr {
173 struct fw_riwrh wrh; /* 0 */
174 union t3_wrid wrid; /* 1 */
179 struct t3_rdma_write_wr {
180 struct fw_riwrh wrh; /* 0 */
181 union t3_wrid wrid; /* 1 */
185 __be64 to_sink; /* 3 */
188 struct t3_sge sgl[T3_MAX_SGE]; /* 5+ */
191 struct t3_rdma_read_wr {
192 struct fw_riwrh wrh; /* 0 */
193 union t3_wrid wrid; /* 1 */
197 __be64 rem_to; /* 3 */
198 __be32 local_stag; /* 4 */
200 __be64 local_to; /* 5 */
204 T3_VA_BASED_TO = 0x0,
205 T3_ZERO_BASED_TO = 0x1
206 } __attribute__ ((packed));
209 T3_MEM_ACCESS_LOCAL_READ = 0x1,
210 T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
211 T3_MEM_ACCESS_REM_READ = 0x4,
212 T3_MEM_ACCESS_REM_WRITE = 0x8
213 } __attribute__ ((packed));
215 struct t3_bind_mw_wr {
216 struct fw_riwrh wrh; /* 0 */
217 union t3_wrid wrid; /* 1 */
218 u16 reserved; /* 2 */
222 __be32 mw_stag; /* 3 */
224 __be64 mw_va; /* 4 */
225 __be32 mr_pbl_addr; /* 5 */
230 struct t3_receive_wr {
231 struct fw_riwrh wrh; /* 0 */
232 union t3_wrid wrid; /* 1 */
233 u8 pagesz[T3_MAX_SGE];
234 __be32 num_sgle; /* 2 */
235 struct t3_sge sgl[T3_MAX_SGE]; /* 3+ */
236 __be32 pbl_addr[T3_MAX_SGE];
239 struct t3_bypass_wr {
241 union t3_wrid wrid; /* 1 */
244 struct t3_modify_qp_wr {
245 struct fw_riwrh wrh; /* 0 */
246 union t3_wrid wrid; /* 1 */
247 __be32 flags; /* 2 */
248 __be32 quiesce; /* 2 */
249 __be32 max_ird; /* 3 */
250 __be32 max_ord; /* 3 */
251 __be64 sge_cmd; /* 4 */
256 enum t3_modify_qp_flags {
257 MODQP_QUIESCE = 0x01,
258 MODQP_MAX_IRD = 0x02,
259 MODQP_MAX_ORD = 0x04,
260 MODQP_WRITE_EC = 0x08,
261 MODQP_READ_EC = 0x10,
266 uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
267 uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
268 uP_RI_MPA_CRC_ENABLE = 0x4,
269 uP_RI_MPA_IETF_ENABLE = 0x8
270 } __attribute__ ((packed));
273 uP_RI_QP_RDMA_READ_ENABLE = 0x01,
274 uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
275 uP_RI_QP_BIND_ENABLE = 0x04,
276 uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
277 uP_RI_QP_STAG0_ENABLE = 0x10
278 } __attribute__ ((packed));
280 struct t3_rdma_init_attr {
288 enum t3_mpa_attrs mpaattrs;
289 enum t3_qp_caps qpcaps;
298 struct t3_rdma_init_wr {
299 struct fw_riwrh wrh; /* 0 */
300 union t3_wrid wrid; /* 1 */
303 __be32 scqid; /* 3 */
305 __be32 rq_addr; /* 4 */
310 __be32 flags; /* bits 31-1 - reservered */
311 /* bit 0 - set if RECV posted */
314 __be64 qp_dma_addr; /* 7 */
315 __be32 qp_dma_size; /* 8 */
324 enum rdma_init_wr_flags {
329 struct t3_send_wr send;
330 struct t3_rdma_write_wr write;
331 struct t3_rdma_read_wr read;
332 struct t3_receive_wr recv;
333 struct t3_local_inv_wr local_inv;
334 struct t3_bind_mw_wr bind;
335 struct t3_bypass_wr bypass;
336 struct t3_rdma_init_wr init;
337 struct t3_modify_qp_wr qp_mod;
338 struct t3_genbit genbit;
342 #define T3_SQ_CQE_FLIT 13
343 #define T3_SQ_COOKIE_FLIT 14
345 #define T3_RQ_COOKIE_FLIT 13
346 #define T3_RQ_CQE_FLIT 14
348 static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
350 return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
353 static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
354 enum t3_wr_flags flags, u8 genbit, u32 tid,
357 wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
358 V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
359 V_FW_RIWR_FLAGS(flags));
361 wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
365 ((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit);
369 * T3 ULP2_TX commands
376 /* T3 MC7 RDMA TPT entry format */
379 TPT_NON_SHARED_MR = 0x0,
382 TPT_MW_RELAXED_PROTECTION = 0x3
391 TPT_LOCAL_READ = 0x8,
392 TPT_LOCAL_WRITE = 0x4,
393 TPT_REMOTE_READ = 0x2,
394 TPT_REMOTE_WRITE = 0x1
398 __be32 valid_stag_pdid;
399 __be32 flags_pagesize_qpid;
401 __be32 rsvd_pbl_addr;
404 __be32 va_low_or_fbo;
406 __be32 rsvd_bind_cnt_or_pstag;
407 __be32 rsvd_pbl_size;
410 #define S_TPT_VALID 31
411 #define V_TPT_VALID(x) ((x) << S_TPT_VALID)
412 #define F_TPT_VALID V_TPT_VALID(1U)
414 #define S_TPT_STAG_KEY 23
415 #define M_TPT_STAG_KEY 0xFF
416 #define V_TPT_STAG_KEY(x) ((x) << S_TPT_STAG_KEY)
417 #define G_TPT_STAG_KEY(x) (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
419 #define S_TPT_STAG_STATE 22
420 #define V_TPT_STAG_STATE(x) ((x) << S_TPT_STAG_STATE)
421 #define F_TPT_STAG_STATE V_TPT_STAG_STATE(1U)
423 #define S_TPT_STAG_TYPE 20
424 #define M_TPT_STAG_TYPE 0x3
425 #define V_TPT_STAG_TYPE(x) ((x) << S_TPT_STAG_TYPE)
426 #define G_TPT_STAG_TYPE(x) (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
429 #define M_TPT_PDID 0xFFFFF
430 #define V_TPT_PDID(x) ((x) << S_TPT_PDID)
431 #define G_TPT_PDID(x) (((x) >> S_TPT_PDID) & M_TPT_PDID)
433 #define S_TPT_PERM 28
434 #define M_TPT_PERM 0xF
435 #define V_TPT_PERM(x) ((x) << S_TPT_PERM)
436 #define G_TPT_PERM(x) (((x) >> S_TPT_PERM) & M_TPT_PERM)
438 #define S_TPT_REM_INV_DIS 27
439 #define V_TPT_REM_INV_DIS(x) ((x) << S_TPT_REM_INV_DIS)
440 #define F_TPT_REM_INV_DIS V_TPT_REM_INV_DIS(1U)
442 #define S_TPT_ADDR_TYPE 26
443 #define V_TPT_ADDR_TYPE(x) ((x) << S_TPT_ADDR_TYPE)
444 #define F_TPT_ADDR_TYPE V_TPT_ADDR_TYPE(1U)
446 #define S_TPT_MW_BIND_ENABLE 25
447 #define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE)
448 #define F_TPT_MW_BIND_ENABLE V_TPT_MW_BIND_ENABLE(1U)
450 #define S_TPT_PAGE_SIZE 20
451 #define M_TPT_PAGE_SIZE 0x1F
452 #define V_TPT_PAGE_SIZE(x) ((x) << S_TPT_PAGE_SIZE)
453 #define G_TPT_PAGE_SIZE(x) (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
455 #define S_TPT_PBL_ADDR 0
456 #define M_TPT_PBL_ADDR 0x1FFFFFFF
457 #define V_TPT_PBL_ADDR(x) ((x) << S_TPT_PBL_ADDR)
458 #define G_TPT_PBL_ADDR(x) (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
461 #define M_TPT_QPID 0xFFFFF
462 #define V_TPT_QPID(x) ((x) << S_TPT_QPID)
463 #define G_TPT_QPID(x) (((x) >> S_TPT_QPID) & M_TPT_QPID)
465 #define S_TPT_PSTAG 0
466 #define M_TPT_PSTAG 0xFFFFFF
467 #define V_TPT_PSTAG(x) ((x) << S_TPT_PSTAG)
468 #define G_TPT_PSTAG(x) (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
470 #define S_TPT_PBL_SIZE 0
471 #define M_TPT_PBL_SIZE 0xFFFFF
472 #define V_TPT_PBL_SIZE(x) ((x) << S_TPT_PBL_SIZE)
473 #define G_TPT_PBL_SIZE(x) (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
494 #define M_CQE_OOO 0x1
495 #define G_CQE_OOO(x) ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
496 #define V_CEQ_OOO(x) ((x)<<S_CQE_OOO)
498 #define S_CQE_QPID 12
499 #define M_CQE_QPID 0x7FFFF
500 #define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
501 #define V_CQE_QPID(x) ((x)<<S_CQE_QPID)
503 #define S_CQE_SWCQE 11
504 #define M_CQE_SWCQE 0x1
505 #define G_CQE_SWCQE(x) ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
506 #define V_CQE_SWCQE(x) ((x)<<S_CQE_SWCQE)
508 #define S_CQE_GENBIT 10
509 #define M_CQE_GENBIT 0x1
510 #define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
511 #define V_CQE_GENBIT(x) ((x)<<S_CQE_GENBIT)
513 #define S_CQE_STATUS 5
514 #define M_CQE_STATUS 0x1F
515 #define G_CQE_STATUS(x) ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
516 #define V_CQE_STATUS(x) ((x)<<S_CQE_STATUS)
519 #define M_CQE_TYPE 0x1
520 #define G_CQE_TYPE(x) ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
521 #define V_CQE_TYPE(x) ((x)<<S_CQE_TYPE)
523 #define S_CQE_OPCODE 0
524 #define M_CQE_OPCODE 0xF
525 #define G_CQE_OPCODE(x) ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
526 #define V_CQE_OPCODE(x) ((x)<<S_CQE_OPCODE)
528 #define SW_CQE(x) (G_CQE_SWCQE(be32_to_cpu((x).header)))
529 #define CQE_OOO(x) (G_CQE_OOO(be32_to_cpu((x).header)))
530 #define CQE_QPID(x) (G_CQE_QPID(be32_to_cpu((x).header)))
531 #define CQE_GENBIT(x) (G_CQE_GENBIT(be32_to_cpu((x).header)))
532 #define CQE_TYPE(x) (G_CQE_TYPE(be32_to_cpu((x).header)))
533 #define SQ_TYPE(x) (CQE_TYPE((x)))
534 #define RQ_TYPE(x) (!CQE_TYPE((x)))
535 #define CQE_STATUS(x) (G_CQE_STATUS(be32_to_cpu((x).header)))
536 #define CQE_OPCODE(x) (G_CQE_OPCODE(be32_to_cpu((x).header)))
538 #define CQE_LEN(x) (be32_to_cpu((x).len))
540 /* used for RQ completion processing */
541 #define CQE_WRID_STAG(x) (be32_to_cpu((x).u.rcqe.stag))
542 #define CQE_WRID_MSN(x) (be32_to_cpu((x).u.rcqe.msn))
544 /* used for SQ completion processing */
545 #define CQE_WRID_SQ_WPTR(x) ((x).u.scqe.wrid_hi)
546 #define CQE_WRID_WPTR(x) ((x).u.scqe.wrid_low)
548 /* generic accessor macros */
549 #define CQE_WRID_HI(x) ((x).u.scqe.wrid_hi)
550 #define CQE_WRID_LOW(x) ((x).u.scqe.wrid_low)
552 #define TPT_ERR_SUCCESS 0x0
553 #define TPT_ERR_STAG 0x1 /* STAG invalid: either the */
554 /* STAG is offlimt, being 0, */
555 /* or STAG_key mismatch */
556 #define TPT_ERR_PDID 0x2 /* PDID mismatch */
557 #define TPT_ERR_QPID 0x3 /* QPID mismatch */
558 #define TPT_ERR_ACCESS 0x4 /* Invalid access right */
559 #define TPT_ERR_WRAP 0x5 /* Wrap error */
560 #define TPT_ERR_BOUND 0x6 /* base and bounds voilation */
561 #define TPT_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */
562 /* shared memory region */
563 #define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */
564 /* shared memory region */
565 #define TPT_ERR_ECC 0x9 /* ECC error detected */
566 #define TPT_ERR_ECC_PSTAG 0xA /* ECC error detected when */
567 /* reading PSTAG for a MW */
569 #define TPT_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */
571 #define TPT_ERR_SWFLUSH 0xC /* SW FLUSHED */
572 #define TPT_ERR_CRC 0x10 /* CRC error */
573 #define TPT_ERR_MARKER 0x11 /* Marker error */
574 #define TPT_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */
575 #define TPT_ERR_OUT_OF_RQE 0x13 /* out of RQE */
576 #define TPT_ERR_DDP_VERSION 0x14 /* wrong DDP version */
577 #define TPT_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */
578 #define TPT_ERR_OPCODE 0x16 /* invalid rdma opcode */
579 #define TPT_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */
580 #define TPT_ERR_MSN 0x18 /* MSN error */
581 #define TPT_ERR_TBIT 0x19 /* tag bit not set correctly */
582 #define TPT_ERR_MO 0x1A /* MO not 0 for TERMINATE */
584 #define TPT_ERR_MSN_GAP 0x1B
585 #define TPT_ERR_MSN_RANGE 0x1C
586 #define TPT_ERR_IRD_OVERFLOW 0x1D
587 #define TPT_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */
589 #define TPT_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */
603 * A T3 WQ implements both the SQ and RQ.
606 union t3_wr *queue; /* DMA accessable memory */
607 dma_addr_t dma_addr; /* DMA address for HW */
608 DECLARE_PCI_UNMAP_ADDR(mapping) /* unmap kruft */
609 u32 error; /* 1 once we go to ERROR */
611 u32 wptr; /* idx to next available WR slot */
612 u32 size_log2; /* total wq size */
613 struct t3_swsq *sq; /* SW SQ */
614 struct t3_swsq *oldest_read; /* tracks oldest pending read */
615 u32 sq_wptr; /* sq_wptr - sq_rptr == count of */
616 u32 sq_rptr; /* pending wrs */
617 u32 sq_size_log2; /* sq size */
618 u64 *rq; /* SW RQ (holds consumer wr_ids */
619 u32 rq_wptr; /* rq_wptr - rq_rptr == count of */
620 u32 rq_rptr; /* pending wrs */
621 u64 *rq_oldest_wr; /* oldest wr on the SW RQ */
622 u32 rq_size_log2; /* rq size */
623 u32 rq_addr; /* rq adapter address */
624 void __iomem *doorbell; /* kernel db */
625 u64 udb; /* user db if any */
634 DECLARE_PCI_UNMAP_ADDR(mapping)
635 struct t3_cqe *queue;
636 struct t3_cqe *sw_queue;
641 #define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
644 static inline void cxio_set_wq_in_error(struct t3_wq *wq)
646 wq->queue->flit[13] = 1;
649 static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
653 cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
654 if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
659 static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
663 if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
664 cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
670 static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
674 if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
675 cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
678 cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
679 if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))