cm_id->context = p;
        p->jiffies = jiffies;
        spin_lock_irq(&priv->lock);
+       if (list_empty(&priv->cm.passive_ids))
+               queue_delayed_work(ipoib_workqueue,
+                                  &priv->cm.stale_task, IPOIB_CM_RX_DELAY);
        list_add(&p->list, &priv->cm.passive_ids);
        spin_unlock_irq(&priv->lock);
-       queue_delayed_work(ipoib_workqueue,
-                          &priv->cm.stale_task, IPOIB_CM_RX_DELAY);
        return 0;
 
 err_rep:
                        if (!list_empty(&p->list))
                                list_move(&p->list, &priv->cm.passive_ids);
                        spin_unlock_irqrestore(&priv->lock, flags);
-                       queue_delayed_work(ipoib_workqueue,
-                                          &priv->cm.stale_task, IPOIB_CM_RX_DELAY);
                }
        }
 
                kfree(p);
                spin_lock_irq(&priv->lock);
        }
+
+       if (!list_empty(&priv->cm.passive_ids))
+               queue_delayed_work(ipoib_workqueue,
+                                  &priv->cm.stale_task, IPOIB_CM_RX_DELAY);
        spin_unlock_irq(&priv->lock);
 }