avoid "Already found the key in hash [CONN_UNUSED_HASH]" messages

Branch HEAD
b=16109
i=johann
i=green

lustre/ChangeLog

@@ -12,6 +12,14 @@ tbd  Sun Microsystems, Inc.
        * RHEL 4 and RHEL 5/SLES 10 clients behaves differently on 'cd' to a
         removed cwd "./" (refer to Bugzilla 14399).
 
+Severity   : minor
+Bugzilla   : 16109
+Frequency  : rare
+Description: avoid Already found the key in hash [CONN_UNUSED_HASH] messages
+Details    : When connection is reused this not moved from CONN_UNUSED_HASH into
+             CONN_USED_HASH and this prodice warning when put connection again
+             in unused hash.
+
 Severity   : enhancement
 Bugzilla   : 16573
 Description: Export bytes_read/bytes_write count on OSC/OST.

lustre/ptlrpc/connection.c

@@ -48,42 +48,54 @@
 
 static spinlock_t conn_lock;
 static struct list_head conn_list;
-static struct list_head conn_unused_list;
 static struct lustre_class_hash_body *conn_hash_body;
 static struct lustre_class_hash_body *conn_unused_hash_body;
 
 extern struct lustre_hash_operations conn_hash_operations;
 
+void ptlrpc_dump_connection(void *obj, void *data)
+{
+        struct ptlrpc_connection *c = obj;
+
+        CERROR("Connection %p/%s has refcount %d (nid=%s->%s)\n",
+                c, c->c_remote_uuid.uuid, atomic_read(&c->c_refcount),
+                libcfs_nid2str(c->c_self),
+                libcfs_nid2str(c->c_peer.nid));
+}
+
 void ptlrpc_dump_connections(void)
 {
-        struct list_head *tmp;
-        struct ptlrpc_connection *c;
         ENTRY;
 
-        list_for_each(tmp, &conn_list) {
-                c = list_entry(tmp, struct ptlrpc_connection, c_link);
-                CERROR("Connection %p/%s has refcount %d (nid=%s->%s)\n",
-                       c, c->c_remote_uuid.uuid, atomic_read(&c->c_refcount),
-                       libcfs_nid2str(c->c_self), 
-                       libcfs_nid2str(c->c_peer.nid));
-        }
+        lustre_hash_iterate_all(conn_hash_body, ptlrpc_dump_connection, NULL);
+
         EXIT;
 }
 
 struct ptlrpc_connection*
 ptlrpc_lookup_conn_locked (lnet_process_id_t peer)
 {
-        struct ptlrpc_connection *c;
+        struct ptlrpc_connection *c = NULL;
+        int rc;
 
         c = lustre_hash_get_object_by_key(conn_hash_body, &peer);
         if (c != NULL)
                 return c;
 
         c = lustre_hash_get_object_by_key(conn_unused_hash_body, &peer);
-        if (c != NULL)
-                return c;
+        if (c != NULL) {
+                lustre_hash_delitem(conn_unused_hash_body, &peer, &c->c_hash);
+                rc = lustre_hash_additem_unique(conn_hash_body, &peer,
+                                                &c->c_hash);
+                if (rc) {
+                        /* can't add - try with new item */
+                        OBD_FREE_PTR(c);
+                        list_del(&c->c_link);
+                        c = NULL;
+                }
+        }
 
-        return NULL;
+        return c;
 }
 
 
@@ -99,15 +111,13 @@ struct ptlrpc_connection *ptlrpc_get_connection(lnet_process_id_t peer,
                libcfs_nid2str(self), libcfs_id2str(peer));
 
         spin_lock(&conn_lock);
-
         c = ptlrpc_lookup_conn_locked(peer);
-        
         spin_unlock(&conn_lock);
 
         if (c != NULL)
                 RETURN (c);
-        
-        OBD_ALLOC(c, sizeof(*c));
+
+        OBD_ALLOC_PTR(c);
         if (c == NULL)
                 RETURN (NULL);
 
@@ -123,18 +133,16 @@ struct ptlrpc_connection *ptlrpc_get_connection(lnet_process_id_t peer,
 
         c2 = ptlrpc_lookup_conn_locked(peer);
         if (c2 == NULL) {
-                list_add(&c->c_link, &conn_list);
                 rc = lustre_hash_additem_unique(conn_hash_body, &peer, 
                                                 &c->c_hash);
                 if (rc != 0) {
-                        list_del(&c->c_link);
                         CERROR("Cannot add connection to conn_hash_body\n");
                         goto out_conn;
                 }
+                list_add(&c->c_link, &conn_list);
         }
 
 out_conn:
-
         spin_unlock(&conn_lock);
 
         if (c2 == NULL && rc == 0)
@@ -166,22 +174,16 @@ int ptlrpc_put_connection(struct ptlrpc_connection *c)
         if (atomic_dec_return(&c->c_refcount) == 1) {
 
                 spin_lock(&conn_lock);
-
                 lustre_hash_delitem(conn_hash_body, &c->c_peer, &c->c_hash);
-                list_del(&c->c_link);
-
-                list_add(&c->c_link, &conn_unused_list);
                 rc = lustre_hash_additem_unique(conn_unused_hash_body, &c->c_peer, 
                                                 &c->c_hash);
+                spin_lock(&conn_lock);
                 if (rc != 0) {
-                        spin_unlock(&conn_lock);
                         CERROR("Cannot hash connection to conn_hash_body\n");
                         GOTO(ret, rc);
                 }
 
-                spin_unlock(&conn_lock);
                 rc = 1;
- 
         } 
 
         if (atomic_read(&c->c_refcount) < 0)
@@ -211,7 +213,6 @@ int ptlrpc_init_connection(void)
         if (rc)
                 GOTO(ret, rc);
 
-        CFS_INIT_LIST_HEAD(&conn_unused_list);
         rc = lustre_hash_init(&conn_unused_hash_body, "CONN_UNUSED_HASH", 
                               128, &conn_hash_operations);
         if (rc)
@@ -234,18 +235,15 @@ void ptlrpc_cleanup_connection(void)
         spin_lock(&conn_lock);
 
         lustre_hash_exit(&conn_unused_hash_body);
-        list_for_each_safe(tmp, pos, &conn_unused_list) {
-                c = list_entry(tmp, struct ptlrpc_connection, c_link);
-                list_del(&c->c_link);
-                OBD_FREE(c, sizeof(*c));
-        }
-
         lustre_hash_exit(&conn_hash_body);
+
         list_for_each_safe(tmp, pos, &conn_list) {
                 c = list_entry(tmp, struct ptlrpc_connection, c_link);
-                CERROR("Connection %p/%s has refcount %d (nid=%s)\n",
-                       c, c->c_remote_uuid.uuid, atomic_read(&c->c_refcount),
-                       libcfs_nid2str(c->c_peer.nid));
+                if (atomic_read(&c->c_refcount))
+                        CERROR("Connection %p/%s has refcount %d (nid=%s)\n",
+                               c, c->c_remote_uuid.uuid,
+                               atomic_read(&c->c_refcount),
+                               libcfs_nid2str(c->c_peer.nid));
                 list_del(&c->c_link);
                 OBD_FREE(c, sizeof(*c));
         }