2 * iSCSI Initiator over iSER Data-Path
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 Mike Christie
7 * Copyright (c) 2005, 2006 Voltaire, Inc. All rights reserved.
8 * maintained by openib-general@openib.org
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
47 * $Id: iscsi_iser.c 6965 2006-05-07 11:36:20Z ogerlitz $
50 #include <linux/types.h>
51 #include <linux/list.h>
52 #include <linux/hardirq.h>
53 #include <linux/kfifo.h>
54 #include <linux/blkdev.h>
55 #include <linux/init.h>
56 #include <linux/ioctl.h>
57 #include <linux/cdev.h>
59 #include <linux/net.h>
60 #include <linux/scatterlist.h>
61 #include <linux/delay.h>
65 #include <asm/uaccess.h>
67 #include <scsi/scsi_cmnd.h>
68 #include <scsi/scsi_device.h>
69 #include <scsi/scsi_eh.h>
70 #include <scsi/scsi_tcq.h>
71 #include <scsi/scsi_host.h>
72 #include <scsi/scsi.h>
73 #include <scsi/scsi_transport_iscsi.h>
75 #include "iscsi_iser.h"
77 static struct scsi_host_template iscsi_iser_sht;
78 static struct iscsi_transport iscsi_iser_transport;
79 static struct scsi_transport_template *iscsi_iser_scsi_transport;
81 static unsigned int iscsi_max_lun = 512;
82 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
84 int iser_debug_level = 0;
86 MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover "
87 "v" DRV_VER " (" DRV_DATE ")");
88 MODULE_LICENSE("Dual BSD/GPL");
89 MODULE_AUTHOR("Alex Nezhinsky, Dan Bar Dov, Or Gerlitz");
91 module_param_named(debug_level, iser_debug_level, int, 0644);
92 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:disabled)");
94 struct iser_global ig;
97 iscsi_iser_recv(struct iscsi_conn *conn,
98 struct iscsi_hdr *hdr, char *rx_data, int rx_data_len)
105 /* verify PDU length */
106 datalen = ntoh24(hdr->dlength);
107 if (datalen != rx_data_len) {
108 printk(KERN_ERR "iscsi_iser: datalen %d (hdr) != %d (IB) \n",
109 datalen, rx_data_len);
110 rc = ISCSI_ERR_DATALEN;
115 ahslen = hdr->hlength * 4;
117 /* verify itt (itt encoding: age+cid+itt) */
118 rc = iscsi_verify_itt(conn, hdr, &ret_itt);
121 rc = iscsi_complete_pdu(conn, hdr, rx_data, rx_data_len);
123 if (rc && rc != ISCSI_ERR_NO_SCSI_CMD)
128 iscsi_conn_failure(conn, rc);
133 * iscsi_iser_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
137 iscsi_iser_cmd_init(struct iscsi_cmd_task *ctask)
139 struct iscsi_iser_conn *iser_conn = ctask->conn->dd_data;
140 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data;
142 iser_ctask->command_sent = 0;
143 iser_ctask->iser_conn = iser_conn;
144 iser_ctask_rdma_init(iser_ctask);
149 * iscsi_mtask_xmit - xmit management(immediate) task
150 * @conn: iscsi connection
151 * @mtask: task management task
154 * The function can return -EAGAIN in which case caller must
155 * call it again later, or recover. '0' return code means successful
160 iscsi_iser_mtask_xmit(struct iscsi_conn *conn,
161 struct iscsi_mgmt_task *mtask)
165 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id, mtask->itt);
167 error = iser_send_control(conn, mtask);
169 /* since iser xmits control with zero copy, mtasks can not be recycled
170 * right after sending them.
171 * The recycling scheme is based on whether a response is expected
172 * - if yes, the mtask is recycled at iscsi_complete_pdu
173 * - if no, the mtask is recycled at iser_snd_completion
175 if (error && error != -ENOBUFS)
176 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
182 iscsi_iser_ctask_xmit_unsol_data(struct iscsi_conn *conn,
183 struct iscsi_cmd_task *ctask)
185 struct iscsi_data hdr;
188 /* Send data-out PDUs while there's still unsolicited data to send */
189 while (ctask->unsol_count > 0) {
190 iscsi_prep_unsolicit_data_pdu(ctask, &hdr);
191 debug_scsi("Sending data-out: itt 0x%x, data count %d\n",
192 hdr.itt, ctask->data_count);
194 /* the buffer description has been passed with the command */
195 /* Send the command */
196 error = iser_send_data_out(conn, ctask, &hdr);
198 ctask->unsol_datasn--;
199 goto iscsi_iser_ctask_xmit_unsol_data_exit;
201 ctask->unsol_count -= ctask->data_count;
202 debug_scsi("Need to send %d more as data-out PDUs\n",
206 iscsi_iser_ctask_xmit_unsol_data_exit:
211 iscsi_iser_ctask_xmit(struct iscsi_conn *conn,
212 struct iscsi_cmd_task *ctask)
214 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data;
217 if (ctask->sc->sc_data_direction == DMA_TO_DEVICE) {
218 BUG_ON(scsi_bufflen(ctask->sc) == 0);
220 debug_scsi("cmd [itt %x total %d imm %d unsol_data %d\n",
221 ctask->itt, scsi_bufflen(ctask->sc),
222 ctask->imm_count, ctask->unsol_count);
225 debug_scsi("ctask deq [cid %d itt 0x%x]\n",
226 conn->id, ctask->itt);
228 /* Send the cmd PDU */
229 if (!iser_ctask->command_sent) {
230 error = iser_send_command(conn, ctask);
232 goto iscsi_iser_ctask_xmit_exit;
233 iser_ctask->command_sent = 1;
236 /* Send unsolicited data-out PDU(s) if necessary */
237 if (ctask->unsol_count)
238 error = iscsi_iser_ctask_xmit_unsol_data(conn, ctask);
240 iscsi_iser_ctask_xmit_exit:
241 if (error && error != -ENOBUFS)
242 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
247 iscsi_iser_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
249 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data;
251 if (iser_ctask->status == ISER_TASK_STATUS_STARTED) {
252 iser_ctask->status = ISER_TASK_STATUS_COMPLETED;
253 iser_ctask_rdma_finalize(iser_ctask);
257 static struct iser_conn *
258 iscsi_iser_ib_conn_lookup(__u64 ep_handle)
260 struct iser_conn *ib_conn;
261 struct iser_conn *uib_conn = (struct iser_conn *)(unsigned long)ep_handle;
263 mutex_lock(&ig.connlist_mutex);
264 list_for_each_entry(ib_conn, &ig.connlist, conn_list) {
265 if (ib_conn == uib_conn) {
266 mutex_unlock(&ig.connlist_mutex);
270 mutex_unlock(&ig.connlist_mutex);
271 iser_err("no conn exists for eph %llx\n",(unsigned long long)ep_handle);
275 static struct iscsi_cls_conn *
276 iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
278 struct iscsi_conn *conn;
279 struct iscsi_cls_conn *cls_conn;
280 struct iscsi_iser_conn *iser_conn;
282 cls_conn = iscsi_conn_setup(cls_session, conn_idx);
285 conn = cls_conn->dd_data;
288 * due to issues with the login code re iser sematics
289 * this not set in iscsi_conn_setup - FIXME
291 conn->max_recv_dlength = 128;
293 iser_conn = kzalloc(sizeof(*iser_conn), GFP_KERNEL);
295 goto conn_alloc_fail;
297 /* currently this is the only field which need to be initiated */
298 rwlock_init(&iser_conn->lock);
300 conn->dd_data = iser_conn;
301 iser_conn->iscsi_conn = conn;
306 iscsi_conn_teardown(cls_conn);
311 iscsi_iser_conn_destroy(struct iscsi_cls_conn *cls_conn)
313 struct iscsi_conn *conn = cls_conn->dd_data;
314 struct iscsi_iser_conn *iser_conn = conn->dd_data;
316 iscsi_conn_teardown(cls_conn);
317 if (iser_conn->ib_conn)
318 iser_conn->ib_conn->iser_conn = NULL;
323 iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
324 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
327 struct iscsi_conn *conn = cls_conn->dd_data;
328 struct iscsi_iser_conn *iser_conn;
329 struct iser_conn *ib_conn;
332 error = iscsi_conn_bind(cls_session, cls_conn, is_leading);
336 /* the transport ep handle comes from user space so it must be
337 * verified against the global ib connections list */
338 ib_conn = iscsi_iser_ib_conn_lookup(transport_eph);
340 iser_err("can't bind eph %llx\n",
341 (unsigned long long)transport_eph);
344 /* binds the iSER connection retrieved from the previously
345 * connected ep_handle to the iSCSI layer connection. exchanges
346 * connection pointers */
347 iser_err("binding iscsi conn %p to iser_conn %p\n",conn,ib_conn);
348 iser_conn = conn->dd_data;
349 ib_conn->iser_conn = iser_conn;
350 iser_conn->ib_conn = ib_conn;
352 conn->recv_lock = &iser_conn->lock;
358 iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn)
360 struct iscsi_conn *conn = cls_conn->dd_data;
363 err = iser_conn_set_full_featured_mode(conn);
367 return iscsi_conn_start(cls_conn);
370 static void iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session)
372 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
374 iscsi_session_teardown(cls_session);
375 scsi_remove_host(shost);
376 iscsi_host_teardown(shost);
377 scsi_host_put(shost);
380 static struct iscsi_cls_session *
381 iscsi_iser_session_create(struct Scsi_Host *shost,
382 uint16_t cmds_max, uint16_t qdepth,
383 uint32_t initial_cmdsn, uint32_t *hostno)
385 struct iscsi_cls_session *cls_session;
386 struct iscsi_session *session;
388 struct iscsi_cmd_task *ctask;
389 struct iscsi_mgmt_task *mtask;
390 struct iscsi_iser_cmd_task *iser_ctask;
391 struct iser_desc *desc;
394 printk(KERN_ERR "iscsi_tcp: invalid shost %d.\n",
399 shost = scsi_host_alloc(&iscsi_iser_sht, 0);
402 shost->transportt = iscsi_iser_scsi_transport;
403 shost->max_lun = iscsi_max_lun;
405 shost->max_channel = 0;
406 shost->max_cmd_len = 16;
408 iscsi_host_setup(shost, qdepth);
410 if (scsi_add_host(shost, NULL))
412 *hostno = shost->host_no;
415 * we do not support setting can_queue cmd_per_lun from userspace yet
416 * because we preallocate so many resources
418 cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,
419 ISCSI_DEF_XMIT_CMDS_MAX,
420 sizeof(struct iscsi_iser_cmd_task),
421 sizeof(struct iser_desc),
425 session = cls_session->dd_data;
427 shost->can_queue = session->cmds_max;
428 /* libiscsi setup itts, data and pool so just set desc fields */
429 for (i = 0; i < session->cmds_max; i++) {
430 ctask = session->cmds[i];
431 iser_ctask = ctask->dd_data;
432 ctask->hdr = (struct iscsi_cmd *)&iser_ctask->desc.iscsi_header;
433 ctask->hdr_max = sizeof(iser_ctask->desc.iscsi_header);
436 for (i = 0; i < session->mgmtpool_max; i++) {
437 mtask = session->mgmt_cmds[i];
438 desc = mtask->dd_data;
439 mtask->hdr = &desc->iscsi_header;
440 desc->data = mtask->data;
446 scsi_remove_host(shost);
448 iscsi_host_teardown(shost);
449 scsi_host_put(shost);
454 iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn,
455 enum iscsi_param param, char *buf, int buflen)
460 case ISCSI_PARAM_MAX_RECV_DLENGTH:
463 case ISCSI_PARAM_HDRDGST_EN:
464 sscanf(buf, "%d", &value);
466 printk(KERN_ERR "DataDigest wasn't negotiated to None");
470 case ISCSI_PARAM_DATADGST_EN:
471 sscanf(buf, "%d", &value);
473 printk(KERN_ERR "DataDigest wasn't negotiated to None");
477 case ISCSI_PARAM_IFMARKER_EN:
478 sscanf(buf, "%d", &value);
480 printk(KERN_ERR "IFMarker wasn't negotiated to No");
484 case ISCSI_PARAM_OFMARKER_EN:
485 sscanf(buf, "%d", &value);
487 printk(KERN_ERR "OFMarker wasn't negotiated to No");
492 return iscsi_set_param(cls_conn, param, buf, buflen);
499 iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
501 struct iscsi_conn *conn = cls_conn->dd_data;
503 stats->txdata_octets = conn->txdata_octets;
504 stats->rxdata_octets = conn->rxdata_octets;
505 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
506 stats->dataout_pdus = conn->dataout_pdus_cnt;
507 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
508 stats->datain_pdus = conn->datain_pdus_cnt; /* always 0 */
509 stats->r2t_pdus = conn->r2t_pdus_cnt; /* always 0 */
510 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
511 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
512 stats->custom_length = 4;
513 strcpy(stats->custom[0].desc, "qp_tx_queue_full");
514 stats->custom[0].value = 0; /* TB iser_conn->qp_tx_queue_full; */
515 strcpy(stats->custom[1].desc, "fmr_map_not_avail");
516 stats->custom[1].value = 0; /* TB iser_conn->fmr_map_not_avail */;
517 strcpy(stats->custom[2].desc, "eh_abort_cnt");
518 stats->custom[2].value = conn->eh_abort_cnt;
519 strcpy(stats->custom[3].desc, "fmr_unalign_cnt");
520 stats->custom[3].value = conn->fmr_unalign_cnt;
524 iscsi_iser_ep_connect(struct sockaddr *dst_addr, int non_blocking,
528 struct iser_conn *ib_conn;
530 err = iser_conn_init(&ib_conn);
534 err = iser_connect(ib_conn, NULL, (struct sockaddr_in *)dst_addr, non_blocking);
536 *ep_handle = (__u64)(unsigned long)ib_conn;
543 iscsi_iser_ep_poll(__u64 ep_handle, int timeout_ms)
545 struct iser_conn *ib_conn = iscsi_iser_ib_conn_lookup(ep_handle);
551 rc = wait_event_interruptible_timeout(ib_conn->wait,
552 ib_conn->state == ISER_CONN_UP,
553 msecs_to_jiffies(timeout_ms));
555 /* if conn establishment failed, return error code to iscsi */
557 (ib_conn->state == ISER_CONN_TERMINATING ||
558 ib_conn->state == ISER_CONN_DOWN))
561 iser_err("ib conn %p rc = %d\n", ib_conn, rc);
564 return 1; /* success, this is the equivalent of POLLOUT */
566 return 0; /* timeout */
568 return rc; /* signal */
572 iscsi_iser_ep_disconnect(__u64 ep_handle)
574 struct iser_conn *ib_conn;
576 ib_conn = iscsi_iser_ib_conn_lookup(ep_handle);
580 iser_err("ib conn %p state %d\n",ib_conn, ib_conn->state);
581 iser_conn_terminate(ib_conn);
584 static struct scsi_host_template iscsi_iser_sht = {
585 .module = THIS_MODULE,
586 .name = "iSCSI Initiator over iSER, v." DRV_VER,
587 .queuecommand = iscsi_queuecommand,
588 .change_queue_depth = iscsi_change_queue_depth,
589 .can_queue = ISCSI_DEF_XMIT_CMDS_MAX - 1,
590 .sg_tablesize = ISCSI_ISER_SG_TABLESIZE,
592 .cmd_per_lun = ISCSI_MAX_CMD_PER_LUN,
593 .eh_abort_handler = iscsi_eh_abort,
594 .eh_device_reset_handler= iscsi_eh_device_reset,
595 .eh_host_reset_handler = iscsi_eh_host_reset,
596 .use_clustering = DISABLE_CLUSTERING,
597 .proc_name = "iscsi_iser",
601 static struct iscsi_transport iscsi_iser_transport = {
602 .owner = THIS_MODULE,
604 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T,
605 .param_mask = ISCSI_MAX_RECV_DLENGTH |
606 ISCSI_MAX_XMIT_DLENGTH |
609 ISCSI_INITIAL_R2T_EN |
614 ISCSI_PDU_INORDER_EN |
615 ISCSI_DATASEQ_INORDER_EN |
617 ISCSI_PERSISTENT_PORT |
618 ISCSI_PERSISTENT_ADDRESS |
619 ISCSI_TARGET_NAME | ISCSI_TPGT |
620 ISCSI_USERNAME | ISCSI_PASSWORD |
621 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
622 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
623 ISCSI_PING_TMO | ISCSI_RECV_TMO,
624 .host_param_mask = ISCSI_HOST_HWADDRESS |
625 ISCSI_HOST_NETDEV_NAME |
626 ISCSI_HOST_INITIATOR_NAME,
627 .conndata_size = sizeof(struct iscsi_conn),
628 .sessiondata_size = sizeof(struct iscsi_session),
629 /* session management */
630 .create_session = iscsi_iser_session_create,
631 .destroy_session = iscsi_iser_session_destroy,
632 /* connection management */
633 .create_conn = iscsi_iser_conn_create,
634 .bind_conn = iscsi_iser_conn_bind,
635 .destroy_conn = iscsi_iser_conn_destroy,
636 .set_param = iscsi_iser_set_param,
637 .get_conn_param = iscsi_conn_get_param,
638 .get_session_param = iscsi_session_get_param,
639 .start_conn = iscsi_iser_conn_start,
640 .stop_conn = iscsi_conn_stop,
641 /* iscsi host params */
642 .get_host_param = iscsi_host_get_param,
643 .set_host_param = iscsi_host_set_param,
645 .send_pdu = iscsi_conn_send_pdu,
646 .get_stats = iscsi_iser_conn_get_stats,
647 .init_cmd_task = iscsi_iser_cmd_init,
648 .xmit_cmd_task = iscsi_iser_ctask_xmit,
649 .xmit_mgmt_task = iscsi_iser_mtask_xmit,
650 .cleanup_cmd_task = iscsi_iser_cleanup_ctask,
652 .session_recovery_timedout = iscsi_session_recovery_timedout,
654 .ep_connect = iscsi_iser_ep_connect,
655 .ep_poll = iscsi_iser_ep_poll,
656 .ep_disconnect = iscsi_iser_ep_disconnect
659 static int __init iser_init(void)
663 iser_dbg("Starting iSER datamover...\n");
665 if (iscsi_max_lun < 1) {
666 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun);
670 memset(&ig, 0, sizeof(struct iser_global));
672 ig.desc_cache = kmem_cache_create("iser_descriptors",
673 sizeof (struct iser_desc),
674 0, SLAB_HWCACHE_ALIGN,
676 if (ig.desc_cache == NULL)
679 /* device init is called only after the first addr resolution */
680 mutex_init(&ig.device_list_mutex);
681 INIT_LIST_HEAD(&ig.device_list);
682 mutex_init(&ig.connlist_mutex);
683 INIT_LIST_HEAD(&ig.connlist);
685 iscsi_iser_scsi_transport = iscsi_register_transport(
686 &iscsi_iser_transport);
687 if (!iscsi_iser_scsi_transport) {
688 iser_err("iscsi_register_transport failed\n");
690 goto register_transport_failure;
695 register_transport_failure:
696 kmem_cache_destroy(ig.desc_cache);
701 static void __exit iser_exit(void)
703 iser_dbg("Removing iSER datamover...\n");
704 iscsi_unregister_transport(&iscsi_iser_transport);
705 kmem_cache_destroy(ig.desc_cache);
708 module_init(iser_init);
709 module_exit(iser_exit);