From 4ac711352b1f85a0d945308c961731ecf9fa8862 Mon Sep 17 00:00:00 2001 From: eeb <eeb> Date: Fri, 27 Feb 2004 19:07:14 +0000 Subject: [PATCH] * Changed obdecho persistent object id to 1; object id 0 is now illegal. * Fixed obdecho persistent object size at 1 MByte. * Changed test_brw data verification tests to work on a fixed 4k block rather than on page size to allow different page sized servers and clients. --- lustre/include/linux/lustre_debug.h | 4 +- lustre/include/linux/obd_echo.h | 7 ++ lustre/obdclass/debug.c | 8 +- lustre/obdecho/echo.c | 131 +++++++++++++++++++--------- lustre/obdecho/echo_client.c | 131 +++++++++++++++++----------- lustre/tests/test_brw.c | 16 ++-- lustre/utils/obd.c | 11 ++- 7 files changed, 198 insertions(+), 110 deletions(-) diff --git a/lustre/include/linux/lustre_debug.h b/lustre/include/linux/lustre_debug.h index 756d32eeb0..669c0e81ef 100644 --- a/lustre/include/linux/lustre_debug.h +++ b/lustre/include/linux/lustre_debug.h @@ -48,6 +48,6 @@ int dump_rniobuf(struct niobuf_remote *rnb); int dump_ioo(struct obd_ioobj *nb); int dump_req(struct ptlrpc_request *req); int dump_obdo(struct obdo *oa); -int page_debug_setup(void *addr, int len, __u64 off, __u64 id); -int page_debug_check(char *who, void *addr, int len, __u64 off, __u64 id); +int block_debug_setup(void *addr, int len, __u64 off, __u64 id); +int block_debug_check(char *who, void *addr, int len, __u64 off, __u64 id); #endif diff --git a/lustre/include/linux/obd_echo.h b/lustre/include/linux/obd_echo.h index 5ff5e6c304..d885579fc9 100644 --- a/lustre/include/linux/obd_echo.h +++ b/lustre/include/linux/obd_echo.h @@ -13,6 +13,13 @@ #define OBD_ECHO_DEVICENAME "obdecho" #define OBD_ECHO_CLIENT_DEVICENAME "echo_client" +/* The persistent object (i.e. actually stores stuff!) */ +#define ECHO_PERSISTENT_OBJID 1ULL +#define ECHO_PERSISTENT_SIZE ((__u64)(1<<20)) + +/* block size to use for data verification */ +#define OBD_ECHO_BLOCK_SIZE (4<<10) + struct ec_object { struct list_head eco_obj_chain; struct obd_device *eco_device; diff --git a/lustre/obdclass/debug.c b/lustre/obdclass/debug.c index 04d17b90bb..fa125c84d3 100644 --- a/lustre/obdclass/debug.c +++ b/lustre/obdclass/debug.c @@ -114,7 +114,7 @@ int dump_req(struct ptlrpc_request *req) */ #define LPDS sizeof(__u64) -int page_debug_setup(void *addr, int len, __u64 off, __u64 id) +int block_debug_setup(void *addr, int len, __u64 off, __u64 id) { LASSERT(addr); @@ -130,7 +130,7 @@ int page_debug_setup(void *addr, int len, __u64 off, __u64 id) return 0; } -int page_debug_check(char *who, void *addr, int end, __u64 off, __u64 id) +int block_debug_check(char *who, void *addr, int end, __u64 off, __u64 id) { __u64 ne_off; int err = 0; @@ -171,5 +171,5 @@ EXPORT_SYMBOL(dump_rniobuf); EXPORT_SYMBOL(dump_ioo); //EXPORT_SYMBOL(dump_req); EXPORT_SYMBOL(dump_obdo); -EXPORT_SYMBOL(page_debug_setup); -EXPORT_SYMBOL(page_debug_check); +EXPORT_SYMBOL(block_debug_setup); +EXPORT_SYMBOL(block_debug_check); diff --git a/lustre/obdecho/echo.c b/lustre/obdecho/echo.c index a9225171af..8ac9b5a28b 100644 --- a/lustre/obdecho/echo.c +++ b/lustre/obdecho/echo.c @@ -51,8 +51,8 @@ #define ECHO_INIT_OBJID 0x1000000000000000ULL #define ECHO_HANDLE_MAGIC 0xabcd0123fedc9876ULL -#define ECHO_OBJECT0_NPAGES 16 -static struct page *echo_object0_pages[ECHO_OBJECT0_NPAGES]; +#define ECHO_PERSISTENT_PAGES (ECHO_PERSISTENT_SIZE/PAGE_SIZE) +static struct page *echo_persistent_pages[ECHO_PERSISTENT_PAGES]; enum { LPROC_ECHO_READ_BYTES = 1, @@ -201,6 +201,62 @@ static int echo_setattr(struct obd_export *exp, struct obdo *oa, return 0; } +static void +echo_page_debug_setup(struct page *page, int rw, obd_id id, + __u64 offset, int len) +{ + int page_offset = offset & (PAGE_SIZE - 1); + char *addr = ((char *)kmap(page)) + page_offset; + + if (len % OBD_ECHO_BLOCK_SIZE != 0) + CERROR("Unexpected block size %d\n", len); + + while (len > 0) { + if (rw & OBD_BRW_READ) + block_debug_setup(addr, OBD_ECHO_BLOCK_SIZE, + offset, id); + else + block_debug_setup(addr, OBD_ECHO_BLOCK_SIZE, + 0xecc0ecc0ecc0ecc0ULL, + 0xecc0ecc0ecc0ecc0ULL); + + addr += OBD_ECHO_BLOCK_SIZE; + offset += OBD_ECHO_BLOCK_SIZE; + len -= OBD_ECHO_BLOCK_SIZE; + } + + kunmap(page); +} + +static int +echo_page_debug_check(struct page *page, obd_id id, + __u64 offset, int len) +{ + int page_offset = offset & (PAGE_SIZE - 1); + char *addr = ((char *)kmap(page)) + page_offset; + int rc = 0; + int rc2; + + if (len % OBD_ECHO_BLOCK_SIZE != 0) + CERROR("Unexpected block size %d\n", len); + + while (len > 0) { + rc2 = block_debug_check("echo", addr, OBD_ECHO_BLOCK_SIZE, + offset, id); + + if (rc2 != 0 && rc == 0) + rc = rc2; + + addr += OBD_ECHO_BLOCK_SIZE; + offset += OBD_ECHO_BLOCK_SIZE; + len -= OBD_ECHO_BLOCK_SIZE; + } + + kunmap(page); + + return (rc); +} + /* This allows us to verify that desc_private is passed unmolested */ #define DESC_PRIV 0x10293847 @@ -233,16 +289,15 @@ int echo_preprw(int cmd, struct obd_export *export, struct obdo *oa, for (i = 0; i < objcount; i++, obj++) { int gfp_mask = (obj->ioo_id & 1) ? GFP_HIGHUSER : GFP_KERNEL; - int isobj0 = obj->ioo_id == 0; - int verify = !isobj0; + int ispersistent = obj->ioo_id == ECHO_PERSISTENT_OBJID; int j; for (j = 0 ; j < obj->ioo_bufcnt ; j++, nb++, r++) { - if (isobj0 && - (nb->offset >> PAGE_SHIFT) < ECHO_OBJECT0_NPAGES) { - r->page = echo_object0_pages[nb->offset >> - PAGE_SHIFT]; + if (ispersistent && + (nb->offset >> PAGE_SHIFT) < ECHO_PERSISTENT_PAGES) { + r->page = echo_persistent_pages[nb->offset >> + PAGE_SHIFT]; /* Take extra ref so __free_pages() can be called OK */ get_page (r->page); } else { @@ -266,22 +321,12 @@ int echo_preprw(int cmd, struct obd_export *export, struct obdo *oa, CDEBUG(D_PAGE, "$$$$ get page %p @ "LPU64" for %d\n", r->page, r->offset, r->len); - if (cmd & OBD_BRW_READ) { - r->rc = r->len; - if (verify) { - page_debug_setup(kmap (r->page), r->len, - r->offset,obj->ioo_id); - kunmap (r->page); - } + if (cmd & OBD_BRW_READ) r->rc = r->len; - } else { - if (verify) { - page_debug_setup(kmap (r->page), r->len, - 0xecc0ecc0ecc0ecc0ULL, - 0xecc0ecc0ecc0ecc0ULL); - kunmap (r->page); - } - } + + if (!ispersistent) + echo_page_debug_setup(r->page, cmd, obj->ioo_id, + r->offset, r->len); } } if (cmd & OBD_BRW_READ) @@ -305,7 +350,7 @@ preprw_cleanup: CERROR("cleaning up %ld pages (%d obdos)\n", (long)(r - res), objcount); while (r-- > res) { kunmap(r->page); - /* NB if this is an 'object0' page, __free_pages will just + /* NB if this is a persistent page, __free_pages will just * lose the extra ref gained above */ __free_pages(r->page, 0); atomic_dec(&obd->u.echo.eo_prep); @@ -344,7 +389,7 @@ int echo_commitrw(int cmd, struct obd_export *export, struct obdo *oa, LASSERT(oti == NULL || oti->oti_handle == (void *)DESC_PRIV); for (i = 0; i < objcount; i++, obj++) { - int verify = obj->ioo_id != 0; + int verify = obj->ioo_id != ECHO_PERSISTENT_OBJID; int j; for (j = 0 ; j < obj->ioo_bufcnt ; j++, r++) { @@ -363,15 +408,15 @@ int echo_commitrw(int cmd, struct obd_export *export, struct obdo *oa, r->page, addr, r->offset); if (verify) { - vrc = page_debug_check("echo", addr, r->len, - r->offset, obj->ioo_id); + vrc = echo_page_debug_check(page, obj->ioo_id, + r->offset, r->len); /* check all the pages always */ if (vrc != 0 && rc == 0) rc = vrc; } kunmap(page); - /* NB see comment above regarding object0 pages */ + /* NB see comment above regarding persistent pages */ __free_pages(page, 0); atomic_dec(&obd->u.echo.eo_prep); } @@ -386,7 +431,7 @@ commitrw_cleanup: while (++r < res + niocount) { struct page *page = r->page; - /* NB see comment above regarding object0 pages */ + /* NB see comment above regarding persistent pages */ __free_pages(page, 0); atomic_dec(&obd->u.echo.eo_prep); } @@ -473,37 +518,37 @@ extern int echo_client_init(void); extern void echo_client_exit(void); static void -echo_object0_pages_fini (void) +echo_persistent_pages_fini (void) { int i; - for (i = 0; i < ECHO_OBJECT0_NPAGES; i++) - if (echo_object0_pages[i] != NULL) { - __free_pages (echo_object0_pages[i], 0); - echo_object0_pages[i] = NULL; + for (i = 0; i < ECHO_PERSISTENT_PAGES; i++) + if (echo_persistent_pages[i] != NULL) { + __free_pages (echo_persistent_pages[i], 0); + echo_persistent_pages[i] = NULL; } } static int -echo_object0_pages_init (void) +echo_persistent_pages_init (void) { struct page *pg; int i; - for (i = 0; i < ECHO_OBJECT0_NPAGES; i++) { - int gfp_mask = (i < ECHO_OBJECT0_NPAGES/2) ? + for (i = 0; i < ECHO_PERSISTENT_PAGES; i++) { + int gfp_mask = (i < ECHO_PERSISTENT_PAGES/2) ? GFP_KERNEL : GFP_HIGHUSER; pg = alloc_pages (gfp_mask, 0); if (pg == NULL) { - echo_object0_pages_fini (); + echo_persistent_pages_fini (); return (-ENOMEM); } memset (kmap (pg), 0, PAGE_SIZE); kunmap (pg); - echo_object0_pages[i] = pg; + echo_persistent_pages[i] = pg; } return (0); @@ -516,9 +561,11 @@ static int __init obdecho_init(void) printk(KERN_INFO "Lustre: Echo OBD driver; info@clusterfs.com\n"); + LASSERT(PAGE_SIZE % OBD_ECHO_BLOCK_SIZE == 0); + lprocfs_init_vars(echo, &lvars); - rc = echo_object0_pages_init (); + rc = echo_persistent_pages_init (); if (rc != 0) goto failed_0; @@ -533,7 +580,7 @@ static int __init obdecho_init(void) class_unregister_type(OBD_ECHO_DEVICENAME); failed_1: - echo_object0_pages_fini (); + echo_persistent_pages_fini (); failed_0: RETURN(rc); } @@ -542,7 +589,7 @@ static void /*__exit*/ obdecho_exit(void) { echo_client_exit(); class_unregister_type(OBD_ECHO_DEVICENAME); - echo_object0_pages_fini (); + echo_persistent_pages_fini (); } MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>"); diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c index 69862b7ec4..f5bcf79362 100644 --- a/lustre/obdecho/echo_client.c +++ b/lustre/obdecho/echo_client.c @@ -274,7 +274,8 @@ echo_get_object (struct ec_object **ecop, struct obd_device *obd, struct ec_object *eco2; int rc; - if ((oa->o_valid & OBD_MD_FLID) == 0) + if ((oa->o_valid & OBD_MD_FLID) == 0 || + oa->o_id == 0) /* disallow use of object id 0 */ { CERROR ("No valid oid\n"); return (-EINVAL); @@ -423,47 +424,66 @@ echo_get_stripe_off_id (struct lov_stripe_md *lsm, obd_off *offp, obd_id *idp) *offp = offset * stripe_size + woffset % stripe_size; } -static void echo_page_debug_setup(struct lov_stripe_md *lsm, - struct page *page, int rw, obd_id id, - obd_off offset, obd_off count) +static void +echo_client_page_debug_setup(struct lov_stripe_md *lsm, + struct page *page, int rw, obd_id id, + obd_off offset, obd_off count) { - void *addr; - obd_off stripe_off; - obd_id stripe_id; + char *addr; + obd_off stripe_off; + obd_id stripe_id; + int delta; - if (id == 0) - return; + /* no partial pages on the client */ + LASSERT(count == PAGE_SIZE); addr = kmap(page); - if (rw == OBD_BRW_WRITE) { - stripe_off = offset; - stripe_id = id; - echo_get_stripe_off_id(lsm, &stripe_off, &stripe_id); - } else { - stripe_off = 0xdeadbeef00c0ffeeULL; - stripe_id = 0xdeadbeef00c0ffeeULL; + for (delta = 0; delta < PAGE_SIZE; delta += OBD_ECHO_BLOCK_SIZE) { + if (rw == OBD_BRW_WRITE) { + stripe_off = offset + delta; + stripe_id = id; + echo_get_stripe_off_id(lsm, &stripe_off, &stripe_id); + } else { + stripe_off = 0xdeadbeef00c0ffeeULL; + stripe_id = 0xdeadbeef00c0ffeeULL; + } + block_debug_setup(addr + delta, OBD_ECHO_BLOCK_SIZE, + stripe_off, stripe_id); } - page_debug_setup(addr, count, stripe_off, stripe_id); kunmap(page); } -static int echo_page_debug_check(struct lov_stripe_md *lsm, - struct page *page, obd_id id, - obd_off offset, obd_off count) +static int +echo_client_page_debug_check(struct lov_stripe_md *lsm, + struct page *page, obd_id id, + obd_off offset, obd_off count) { - obd_off stripe_off = offset; - obd_id stripe_id = id; - void *addr; - int rc; + obd_off stripe_off; + obd_id stripe_id; + char *addr; + int delta; + int rc; + int rc2; - if (id == 0) - return 0; + /* no partial pages on the client */ + LASSERT(count == PAGE_SIZE); addr = kmap(page); - echo_get_stripe_off_id (lsm, &stripe_off, &stripe_id); - rc = page_debug_check("test_brw", addr, count, stripe_off, stripe_id); + + for (rc = delta = 0; delta < PAGE_SIZE; delta += OBD_ECHO_BLOCK_SIZE) { + stripe_off = offset + delta; + stripe_id = id; + echo_get_stripe_off_id (lsm, &stripe_off, &stripe_id); + + rc2 = block_debug_check("test_brw", + addr + delta, OBD_ECHO_BLOCK_SIZE, + stripe_off, stripe_id); + if (rc2 != 0) + rc = rc2; + } + kunmap(page); return rc; } @@ -482,10 +502,11 @@ static int echo_client_kbrw(struct obd_device *obd, int rw, struct obdo *oa, int verify = 0; int gfp_mask; - /* oa_id == 0 => speed test (no verification) else... - * oa & 1 => use HIGHMEM - */ - gfp_mask = ((oa->o_id & 1) == 0) ? GFP_KERNEL : GFP_HIGHUSER; + /* oa_id == ECHO_PERSISTENT_OBJID => speed test (no verification). + * oa & 1 => use HIGHMEM */ + + verify = (oa->o_id) != ECHO_PERSISTENT_OBJID; + gfp_mask = ((oa->o_id & 2) == 0) ? GFP_KERNEL : GFP_HIGHUSER; LASSERT(rw == OBD_BRW_WRITE || rw == OBD_BRW_READ); @@ -517,15 +538,16 @@ static int echo_client_kbrw(struct obd_device *obd, int rw, struct obdo *oa, pgp->off = off; pgp->flag = 0; - echo_page_debug_setup(lsm, pgp->pg, rw, oa->o_id, off, - pgp->count); + if (verify) + echo_client_page_debug_setup(lsm, pgp->pg, rw, + oa->o_id, off, pgp->count); } rc = obd_brw(rw, ec->ec_exp, oa, lsm, npages, pga, oti); out: - if (rc == 0 && rw == OBD_BRW_READ) - verify = 1; + if (rc != 0 || rw != OBD_BRW_READ) + verify = 0; for (i = 0, pgp = pga; i < npages; i++, pgp++) { if (pgp->pg == NULL) @@ -533,8 +555,8 @@ static int echo_client_kbrw(struct obd_device *obd, int rw, struct obdo *oa, if (verify) { int vrc; - vrc = echo_page_debug_check(lsm, pgp->pg, oa->o_id, - pgp->off, pgp->count); + vrc = echo_client_page_debug_check(lsm, pgp->pg, oa->o_id, + pgp->off, pgp->count); if (vrc != 0 && rc == 0) rc = vrc; } @@ -698,10 +720,11 @@ static void ec_ap_completion(void *data, int cmd, int rc) return; eas = eap->eap_eas; - if (cmd == OBD_BRW_READ) - echo_page_debug_check(eas->eas_lsm, eap->eap_page, - eas->eas_oa.o_id, eap->eap_off, - PAGE_SIZE); + if (cmd == OBD_BRW_READ && + eas->eas_oa.o_id != ECHO_PERSISTENT_OBJID) + echo_client_page_debug_check(eas->eas_lsm, eap->eap_page, + eas->eas_oa.o_id, eap->eap_off, + PAGE_SIZE); spin_lock_irqsave(&eas->eas_lock, flags); if (rc && !eas->eas_rc) @@ -823,9 +846,10 @@ static int echo_client_async_page(struct obd_export *exp, int rw, break; } - if (rw == OBD_BRW_WRITE) - echo_page_debug_setup(lsm, eap->eap_page, rw, oa->o_id, - eap->eap_off, PAGE_SIZE); + if (oa->o_id != ECHO_PERSISTENT_OBJID) + echo_client_page_debug_setup(lsm, eap->eap_page, rw, + oa->o_id, + eap->eap_off, PAGE_SIZE); /* always asserts urgent, which isn't quite right */ rc = obd_queue_async_io(exp, lsm, NULL, eap->eap_cookie, @@ -925,14 +949,19 @@ static int echo_client_prep_commit(struct obd_export *exp, int rw, if (page == NULL && lnb[i].rc == 0) continue; + if (oa->o_id == ECHO_PERSISTENT_OBJID) + continue; + if (rw == OBD_BRW_WRITE) - echo_page_debug_setup(lsm, page, rw, oa->o_id, - rnb[i].offset, - rnb[i].len); + echo_client_page_debug_setup(lsm, page, rw, + oa->o_id, + rnb[i].offset, + rnb[i].len); else - echo_page_debug_check(lsm, page, oa->o_id, - rnb[i].offset, - rnb[i].len); + echo_client_page_debug_check(lsm, page, + oa->o_id, + rnb[i].offset, + rnb[i].len); } ret = obd_commitrw(rw, exp, oa, 1, &ioo, npages, lnb, oti); diff --git a/lustre/tests/test_brw.c b/lustre/tests/test_brw.c index f242eba856..54126a0fbe 100644 --- a/lustre/tests/test_brw.c +++ b/lustre/tests/test_brw.c @@ -31,7 +31,7 @@ #define WRITE 2 #define LPDS sizeof(__u64) -int page_debug_setup(void *addr, int len, __u64 off, __u64 id) +int block_debug_setup(void *addr, int len, __u64 off, __u64 id) { off = cpu_to_le64(off); id = cpu_to_le64(id); @@ -45,7 +45,7 @@ int page_debug_setup(void *addr, int len, __u64 off, __u64 id) return 0; } -int page_debug_check(char *who, void *addr, int size, __u64 off, __u64 id) +int block_debug_check(char *who, void *addr, int size, __u64 off, __u64 id) { __u64 ne_off; int err = 0; @@ -181,14 +181,14 @@ int main(int argc, char **argv) int i; for (i = 0; i < len; i += st.st_blksize) - page_debug_setup(buf + i, st.st_blksize, offset + i, - objid); + block_debug_setup(buf + i, st.st_blksize, + offset + i, objid); rc = write(fd, buf, len); for (i = 0; i < len; i += st.st_blksize) { - if (page_debug_check("write", buf + i, st.st_blksize, - offset + i, objid)) + if (block_debug_check("write", buf + i, st.st_blksize, + offset + i, objid)) return 10; } @@ -216,8 +216,8 @@ int main(int argc, char **argv) } for (i = 0; i < len; i += st.st_blksize) { - if (page_debug_check("read", buf + i, st.st_blksize, - offset + i, objid)) + if (block_debug_check("read", buf + i, st.st_blksize, + offset + i, objid)) return 11; } } diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index ad043aaed0..e4ef0a8530 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -1242,6 +1242,7 @@ int jt_obd_test_brw(int argc, char **argv) struct timeval start, next_time; __u64 count, next_count, len, thr_offset = 0, objid = 3; int write = 0, verbose = 1, cmd, i, rc = 0, pages = 1; + int repeat_offset = 0; char *end; if (argc < 2 || argc > 7) { @@ -1267,8 +1268,11 @@ int jt_obd_test_brw(int argc, char **argv) if (argc >= 3) { if (argv[2][0] == 'w' || argv[2][0] == '1') write = 1; - else if (argv[2][0] == 'r' || argv[2][0] == '0') - write = 0; + /* else it's a read */ + + if (argv[2][0] != 0 && + argv[2][1] == 'r') + repeat_offset = 1; } if (argc >= 4) { @@ -1366,7 +1370,8 @@ int jt_obd_test_brw(int argc, char **argv) printf("%s: %s number %dx%d\n", jt_cmdname(argv[0]), write ? "write" : "read", i, pages); - data.ioc_offset += len; + if (!repeat_offset) + data.ioc_offset += len; } if (!rc) { -- GitLab