From d52ea68f58b41d059dadffadb6420fd8ec7788da Mon Sep 17 00:00:00 2001
From: kalpak <kalpak>
Date: Wed, 17 Oct 2007 07:54:00 +0000
Subject: [PATCH] b=13128 i=adilger, johann
Enhance lfs find by adding filetype and filesize parameters. Also multiple OBDs can now be specified for the --obd option.
---
lustre/ChangeLog | 6 +
lustre/include/lustre/liblustreapi.h | 17 +-
lustre/utils/lfs.c | 130 +++++++++++----
lustre/utils/liblustreapi.c | 231 ++++++++++++++++++++++-----
4 files changed, 304 insertions(+), 80 deletions(-)
diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index c8c04a7978..23f8e54f22 100644
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -14,6 +14,12 @@ tbd Cluster File Systems, Inc. <info@clusterfs.com>
* Recommended e2fsprogs version: 1.40.2-cfs1
* Note that reiserfs quotas are disabled on SLES 10 in this kernel.
+Severity : enhancement
+Bugzilla : 13128
+Description: add --type and --size parameters to lfs find
+Details : Enhance lfs find by adding filetype and filesize parameters. Also
+ multiple OBDs can now be specified for the --obd option.
+
Severity : enhancement
Bugzilla : 11270
Description: eliminate client locks in face of contention
diff --git a/lustre/include/lustre/liblustreapi.h b/lustre/include/lustre/liblustreapi.h
index a1a70f27b8..095446d9af 100644
--- a/lustre/include/lustre/liblustreapi.h
+++ b/lustre/include/lustre/liblustreapi.h
@@ -8,6 +8,12 @@
#include <lustre/lustre_user.h>
+/* Initially allocate for these many OSTs, realloc if needed */
+#define INIT_ALLOC_NUM_OSTS 1024
+
+/* Maximum number of osts that can be specified to lfs find */
+#define FIND_MAX_OSTS 1024
+
typedef void (*llapi_cb_t)(char *obd_type_name, char *obd_name, char *obd_uuid, void *args);
/* liblustreapi.c */
@@ -30,6 +36,10 @@ struct find_param {
int csign;
int msign;
int type;
+ unsigned long long size;
+ int size_sign;
+ unsigned long long size_units;
+ int size_check;
unsigned zeroend:1, recursive:1, got_uuids:1, obds_printed:1, exclude_pattern:1, exclude_type:1;
@@ -42,7 +52,10 @@ struct find_param {
char *print_fmt;
struct obd_uuid *obduuid;
- int obdindex;
+ int num_obds;
+ int num_alloc_obds;
+ int obdindex;
+ int *obdindexes;
int lumlen;
struct lov_user_mds_data *lmd;
@@ -63,6 +76,8 @@ extern int llapi_target_check(int num_types, char **obd_types, char *dir);
extern int llapi_catinfo(char *dir, char *keyword, char *node_name);
extern int llapi_lov_get_uuids(int fd, struct obd_uuid *uuidp, int *ost_count);
extern int llapi_is_lustre_mnttype(const char *type);
+extern int parse_size(char *optarg, unsigned long long *size,
+ unsigned long long *size_units);
struct mntent;
#define HAVE_LLAPI_IS_LUSTRE_MNT
extern int llapi_is_lustre_mnt(struct mntent *mnt);
diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 945737467b..98ff908105 100644
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -147,7 +147,7 @@ static int lfs_setstripe(int argc, char **argv)
{
char *fname;
int result;
- unsigned long st_size;
+ unsigned long long st_size;
int st_offset, st_count;
char *end;
int c;
@@ -155,6 +155,7 @@ static int lfs_setstripe(int argc, char **argv)
char *stripe_size_arg = NULL;
char *stripe_off_arg = NULL;
char *stripe_count_arg = NULL;
+ unsigned long long size_units;
struct option long_opts[] = {
{"size", required_argument, 0, 's'},
@@ -238,25 +239,11 @@ static int lfs_setstripe(int argc, char **argv)
/* get the stripe size */
if (stripe_size_arg != NULL) {
- st_size = strtoul(stripe_size_arg, &end, 0);
- if (*end != '\0') {
- if ((*end == 'k' || *end == 'K') &&
- *(end+1) == '\0' &&
- (st_size & (~0UL << (32 - 10))) == 0) {
- st_size <<= 10;
- } else if ((*end == 'm' || *end == 'M') &&
- *(end+1) == '\0' &&
- (st_size & (~0UL << (32 - 20))) == 0) {
- st_size <<= 20;
- } else if ((*end == 'g' || *end == 'G') &&
- *(end+1) == '\0' &&
- (st_size & (~0UL << (32 - 30))) == 0) {
- st_size <<= 30;
- } else {
- fprintf(stderr, "error: %s: bad stripe size '%s'\n",
- argv[0], stripe_size_arg);
- return CMD_HELP;
- }
+ result = parse_size(stripe_size_arg, &st_size, &size_units);
+ if (result) {
+ fprintf(stderr,"error: bad size '%s'\n",
+ stripe_size_arg);
+ return result;
}
}
/* get the stripe offset */
@@ -317,7 +304,7 @@ static int lfs_find(int argc, char **argv)
int c, ret;
time_t t;
struct find_param param = { .maxdepth = -1 };
- char timestr[1024];
+ char str[1024];
struct option long_opts[] = {
/* New find options. */
{"atime", required_argument, 0, 'A'},
@@ -333,6 +320,7 @@ static int lfs_find(int argc, char **argv)
/* Old find options. */
{"quiet", no_argument, 0, 'q'},
{"recursive", no_argument, 0, 'r'},
+ {"size", required_argument, 0, 's'},
{"type", required_argument, 0, 't'},
{"verbose", no_argument, 0, 'v'},
{0, 0, 0, 0}
@@ -346,7 +334,7 @@ static int lfs_find(int argc, char **argv)
time(&t);
- while ((c = getopt_long_only(argc, argv, "-A:C:D:M:n:PpO:qrt:v",
+ while ((c = getopt_long_only(argc, argv, "-A:C:D:M:n:PpO:qrs:t:v",
long_opts, NULL)) >= 0) {
xtime = NULL;
xsign = NULL;
@@ -397,10 +385,10 @@ static int lfs_find(int argc, char **argv)
else if (optarg[0] == '+')
optarg[0] = '-';
else {
- timestr[0] = '-';
- timestr[1] = '\0';
- strcat(timestr, optarg);
- optarg = timestr;
+ str[0] = '-';
+ str[1] = '\0';
+ strcat(str, optarg);
+ optarg = str;
}
}
ret = set_time(&t, xtime, optarg);
@@ -418,15 +406,39 @@ static int lfs_find(int argc, char **argv)
param.pattern = (char *)optarg;
param.exclude_pattern = !!neg_opt;
break;
- case 'O':
- if (param.obduuid) {
- fprintf(stderr,
- "error: %s: only one obduuid allowed",
- argv[0]);
- return CMD_HELP;
+ case 'O': {
+ char *buf, *token, *next, *p;
+ int len;
+
+ len = strlen((char *)optarg);
+ buf = malloc(len+1);
+ if (buf == NULL)
+ return -ENOMEM;
+ strcpy(buf, (char *)optarg);
+
+ if (param.num_alloc_obds == 0) {
+ param.obduuid = (struct obd_uuid *)malloc(FIND_MAX_OSTS *
+ sizeof(struct obd_uuid));
+ if (param.obduuid == NULL)
+ return -ENOMEM;
+ param.num_alloc_obds = INIT_ALLOC_NUM_OSTS;
}
- param.obduuid = (struct obd_uuid *)optarg;
+
+ for (token = buf; token && *token; token = next) {
+ p = strchr(token, ',');
+ next = 0;
+ if (p) {
+ *p = 0;
+ next = p+1;
+ }
+ strcpy((char *)¶m.obduuid[param.num_obds++].uuid,
+ token);
+ }
+
+ if (buf)
+ free(buf);
break;
+ }
case 'p':
new_fashion = 1;
param.zeroend = 1;
@@ -460,6 +472,34 @@ static int lfs_find(int argc, char **argv)
return CMD_HELP;
};
break;
+ case 's':
+ if (neg_opt) {
+ if (optarg[0] == '-')
+ optarg[0] = '+';
+ else if (optarg[0] == '+')
+ optarg[0] = '-';
+ else {
+ str[0] = '-';
+ str[1] = '\0';
+ strcat(str, optarg);
+ optarg = str;
+ }
+ }
+ if (optarg[0] == '+')
+ param.size_sign = -1;
+ else if (optarg[0] == '-')
+ param.size_sign = +1;
+
+ if (param.size_sign)
+ optarg++;
+ ret = parse_size(optarg, ¶m.size,¶m.size_units);
+ if (ret) {
+ fprintf(stderr,"error: bad size '%s'\n",
+ optarg);
+ return ret;
+ }
+ param.size_check = 1;
+ break;
case 'v':
new_fashion = 0;
param.verbose++;
@@ -507,6 +547,9 @@ static int lfs_find(int argc, char **argv)
fprintf(stderr, "error: %s failed for %s.\n",
argv[0], argv[optind - 1]);
+ if (param.obduuid && param.num_alloc_obds)
+ free(param.obduuid);
+
return ret;
}
@@ -1506,7 +1549,7 @@ static void print_mds_quota(char *mnt, struct if_quotactl *qctl)
static void print_lov_quota(char *mnt, struct if_quotactl *qctl)
{
DIR *dir;
- struct obd_uuid uuids[1024], *uuidp;
+ struct obd_uuid *uuids = NULL, *uuidp;
int obdcount = 1024;
int i, rc;
@@ -1516,9 +1559,26 @@ static void print_lov_quota(char *mnt, struct if_quotactl *qctl)
return;
}
+ uuids = (struct obd_uuid *)malloc(INIT_ALLOC_NUM_OSTS *
+ sizeof(struct obd_uuid));
+ if (uuids == NULL)
+ goto out;
+
+retry_get_uuids:
rc = llapi_lov_get_uuids(dirfd(dir), uuids, &obdcount);
if (rc != 0) {
- fprintf(stderr, "get ost uuid failed: %s\n", strerror(errno));
+ struct obd_uuid *uuids_temp;
+
+ if (rc == -EOVERFLOW) {
+ uuids_temp = realloc(uuids, obdcount *
+ sizeof(struct obd_uuid));
+ if (uuids_temp != NULL)
+ goto retry_get_uuids;
+ else
+ rc = -ENOMEM;
+ }
+
+ fprintf(stderr, "get ost uuid failed: %s\n", strerror(rc));
goto out;
}
diff --git a/lustre/utils/liblustreapi.c b/lustre/utils/liblustreapi.c
index 18f5a4bf67..07610a7f53 100644
--- a/lustre/utils/liblustreapi.c
+++ b/lustre/utils/liblustreapi.c
@@ -67,6 +67,56 @@ static void err_msg(char *fmt, ...)
fprintf(stderr, ": %s (%d)\n", strerror(tmp_errno), tmp_errno);
}
+int parse_size(char *optarg, unsigned long long *size,
+ unsigned long long *size_units)
+{
+ char *end;
+
+ *size = strtoul(optarg, &end, 0);
+
+ if (*end != '\0') {
+ if ((*end == 'b') && *(end+1) == '\0' &&
+ (*size & (~0ULL << (64 - 9))) == 0) {
+ *size <<= 9;
+ *size_units = 1 << 9;
+ } else if ((*end == 'k' || *end == 'K') &&
+ *(end+1) == '\0' && (*size &
+ (~0ULL << (64 - 10))) == 0) {
+ *size <<= 10;
+ *size_units = 1 << 10;
+ } else if ((*end == 'm' || *end == 'M') &&
+ *(end+1) == '\0' && (*size &
+ (~0ULL << (64 - 20))) == 0) {
+ *size <<= 20;
+ *size_units = 1 << 20;
+ } else if ((*end == 'g' || *end == 'G') &&
+ *(end+1) == '\0' && (*size &
+ (~0ULL << (64 - 30))) == 0) {
+ *size <<= 30;
+ *size_units = 1 << 30;
+ } else if ((*end == 't' || *end == 'T') &&
+ *(end+1) == '\0' && (*size &
+ (~0ULL << (64 - 40))) == 0) {
+ *size <<= 40;
+ *size_units = 1ULL << 40;
+ } else if ((*end == 'p' || *end == 'P') &&
+ *(end+1) == '\0' && (*size &
+ (~0ULL << (64 - 50))) == 0) {
+ *size <<= 50;
+ *size_units = 1ULL << 50;
+ } else if ((*end == 'e' || *end == 'E') &&
+ *(end+1) == '\0' && (*size &
+ (~0ULL << (64 - 60))) == 0) {
+ *size <<= 60;
+ *size_units = 1ULL << 60;
+ } else {
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
int llapi_file_open(const char *name, int flags, int mode,
unsigned long stripe_size, int stripe_offset,
int stripe_count, int stripe_pattern)
@@ -174,16 +224,26 @@ static int common_param_init(struct find_param *param)
}
param->got_uuids = 0;
+ param->obdindexes = NULL;
param->obdindex = OBD_NOT_FOUND;
return 0;
}
static void find_param_fini(struct find_param *param)
{
+ if (param->obdindexes)
+ free(param->obdindexes);
+
if (param->lmd)
free(param->lmd);
}
+/*
+ * If uuidp is NULL, return the number of available obd uuids.
+ * If uuidp is non-NULL, then it will return the uuids of the obds. If
+ * there are more OSTs then allocated to uuidp, then an error is returned with
+ * the ost_count set to number of available obd uuids.
+ */
int llapi_lov_get_uuids(int fd, struct obd_uuid *uuidp, int *ost_count)
{
char lov_name[sizeof(struct obd_uuid)];
@@ -209,19 +269,26 @@ int llapi_lov_get_uuids(int fd, struct obd_uuid *uuidp, int *ost_count)
return rc;
}
- while ((fgets(buf, sizeof(buf), fp) != NULL) && index < *ost_count) {
- if (sscanf(buf, "%d: %s", &index, uuidp[index].uuid) < 2)
- break;
+ while (fgets(buf, sizeof(buf), fp) != NULL) {
+ if (uuidp && (index < *ost_count)) {
+ if (sscanf(buf, "%d: %s", &index, uuidp[index].uuid) <2)
+ break;
+ }
index++;
}
fclose(fp);
- *ost_count = index;
+ if (uuidp && (index >= *ost_count))
+ return -EOVERFLOW;
+
+ *ost_count = index;
return rc;
}
-static int setup_obd_uuids(DIR *dir, char *dname, struct find_param *param)
+/* Here, param->obduuid points to a single obduuid, the index of which is
+ * returned in param->obdindex */
+static int setup_obd_uuid(DIR *dir, char *dname, struct find_param *param)
{
char uuid[sizeof(struct obd_uuid)];
char buf[1024];
@@ -284,6 +351,66 @@ static int setup_obd_uuids(DIR *dir, char *dname, struct find_param *param)
return (rc);
}
+/* In this case, param->obduuid will be an array of obduuids and
+ * obd index for all these obduuids will be returned in
+ * param->obdindexes */
+static int setup_obd_indexes(DIR *dir, struct find_param *param)
+{
+ struct obd_uuid *uuids = NULL;
+ int obdcount = INIT_ALLOC_NUM_OSTS;
+ int ret, obd_valid = 0, obdnum, i;
+
+ uuids = (struct obd_uuid *)malloc(INIT_ALLOC_NUM_OSTS *
+ sizeof(struct obd_uuid));
+ if (uuids == NULL)
+ return -ENOMEM;
+
+retry_get_uuids:
+ ret = llapi_lov_get_uuids(dirfd(dir), uuids,
+ &obdcount);
+ if (ret) {
+ struct obd_uuid *uuids_temp;
+
+ if (ret == -EOVERFLOW) {
+ uuids_temp = realloc(uuids, obdcount *
+ sizeof(struct obd_uuid));
+ if (uuids_temp != NULL)
+ goto retry_get_uuids;
+ else
+ ret = -ENOMEM;
+ }
+
+ fprintf(stderr, "get ost uuid failed: %s\n", strerror(errno));
+ return ret;
+ }
+
+ param->obdindexes = malloc(param->num_obds * sizeof(param->obdindex));
+ if (param->obdindexes == NULL)
+ return -ENOMEM;
+
+ for (obdnum = 0; obdnum < param->num_obds; obdnum++) {
+ for (i = 0; i <= obdcount; i++) {
+ if (strcmp((char *)¶m->obduuid[obdnum].uuid,
+ (char *)&uuids[i]) == 0) {
+ param->obdindexes[obdnum] = i;
+ obd_valid++;
+ break;
+ }
+ }
+ if (i == obdcount)
+ param->obdindexes[obdnum] = OBD_NOT_FOUND;
+ }
+
+ if (obd_valid == 0)
+ param->obdindex = OBD_NOT_FOUND;
+ else
+ param->obdindex = obd_valid;
+
+ param->got_uuids = 1;
+
+ return 0;
+}
+
void lov_dump_user_lmm_v1(struct lov_user_md_v1 *lum, char *path, int is_dir,
int obdindex, int quiet, int header, int body)
{
@@ -613,12 +740,12 @@ err:
return ret;
}
-/* Check if the file time matches 1 of the given criteria (e.g. --atime +/-N).
+/* Check if the value matches 1 of the given criteria (e.g. --atime +/-N).
* @mds indicates if this is MDS timestamps and there are attributes on OSTs.
*
* The result is -1 if it does not match, 0 if not yet clear, 1 if matches.
- * The table bolow gives the answers for the specified parameters (time and
- * sign), 1st column is the answer for the MDS time, the 2nd is for the OST:
+ * The table below gives the answers for the specified parameters (value and
+ * sign), 1st column is the answer for the MDS value, the 2nd is for the OST:
* --------------------------------------
* 1 | file > limit; sign > 0 | -1 / -1 |
* 2 | file = limit; sign > 0 | ? / 1 |
@@ -630,18 +757,20 @@ err:
* 8 | file = limit; sign < 0 | ? / -1 |
* 9 | file < limit; sign < 0 | ? / -1 |
* --------------------------------------
- * Note: 5th actually means that the file time stamp is within the interval
- * (limit - 24hours, limit]. */
-static int find_time_cmp(time_t file, time_t limit, int sign, int mds) {
+ * Note: 5th actually means that the value is within the interval
+ * (limit - margin, limit]. */
+static int find_value_cmp(unsigned int file, unsigned int limit, int sign,
+ unsigned long long margin, int mds)
+{
if (sign > 0) {
- if (file <= limit)
+ if (file < limit)
return mds ? 0 : 1;
}
if (sign == 0) {
- if (file <= limit && file + 24 * 60 * 60 > limit)
+ if (file <= limit && file + margin > limit)
return mds ? 0 : 1;
- if (file + 24 * 60 * 60 <= limit)
+ if (file + margin <= limit)
return mds ? 0 : -1;
}
@@ -669,16 +798,16 @@ static int find_time_check(lstat_t *st, struct find_param *param, int mds)
/* Check if file is accepted. */
if (param->atime) {
- ret = find_time_cmp(st->st_atime, param->atime,
- param->asign, mds);
+ ret = find_value_cmp(st->st_atime, param->atime,
+ param->asign, 24 * 60 * 60, mds);
if (ret < 0)
return ret;
rc = ret;
}
if (param->mtime) {
- ret = find_time_cmp(st->st_mtime, param->mtime,
- param->msign, mds);
+ ret = find_value_cmp(st->st_mtime, param->mtime,
+ param->msign, 24 * 60 * 60, mds);
if (ret < 0)
return ret;
@@ -689,8 +818,8 @@ static int find_time_check(lstat_t *st, struct find_param *param, int mds)
}
if (param->ctime) {
- ret = find_time_cmp(st->st_ctime, param->ctime,
- param->csign, mds);
+ ret = find_value_cmp(st->st_ctime, param->ctime,
+ param->csign, 24 * 60 * 60, mds);
if (ret < 0)
return ret;
@@ -746,6 +875,18 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
goto decided;
}
+ /* Check the time on mds. */
+ if (!decision) {
+ int for_mds;
+
+ for_mds = lustre_fs ? (S_ISREG(st->st_mode) &&
+ param->lmd->lmd_lmm.lmm_stripe_count)
+ : 0;
+ decision = find_time_check(st, param, for_mds);
+ if (decision == -1)
+ goto decided;
+ }
+
/* See if we can check the file type from the dirent. */
if (param->type && de != NULL && de->d_type != DT_UNKNOWN &&
de->d_type <= DT_MAX) {
@@ -761,7 +902,8 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
/* If a time or OST should be checked, the decision is not taken yet. */
- if (param->atime || param->ctime || param->mtime || param->obduuid)
+ if (param->atime || param->ctime || param->mtime || param->obduuid ||
+ param->size_check)
decision = 0;
/* Request MDS for the stat info. */
@@ -819,9 +961,10 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
}
if (lustre_fs && !param->got_uuids) {
- ret = setup_obd_uuids(dir ? dir : parent, path, param);
+ ret = setup_obd_indexes(dir ? dir : parent, param);
if (ret)
return ret;
+
param->st_dev = st->st_dev;
} else if (!lustre_fs && param->got_uuids) {
/* A lustre/non-lustre mount point is crossed. */
@@ -832,46 +975,40 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
/* If an OBD UUID is specified but no one matches, skip this file. */
if (param->obduuid && param->obdindex == OBD_NOT_FOUND)
- decision = -1;
+ goto decided;
/* If a OST UUID is given, and some OST matches, check it here. */
- if (decision != -1 && param->obdindex != OBD_NOT_FOUND) {
+ if (param->obdindex != OBD_NOT_FOUND) {
if (!S_ISREG(st->st_mode))
goto decided;
/* Only those files should be accepted, which have a
* stripe on the specified OST. */
if (!param->lmd->lmd_lmm.lmm_stripe_count) {
- decision = -1;
+ goto decided;
} else {
- int i;
+ int i, j;
for (i = 0;
i < param->lmd->lmd_lmm.lmm_stripe_count; i++) {
- if (param->obdindex ==
- param->lmd->lmd_lmm.lmm_objects[i].l_ost_idx)
- break;
+ for (j = 0; j < param->num_obds; j++) {
+ if (param->obdindexes[j] ==
+ param->lmd->lmd_lmm.lmm_objects[i].l_ost_idx)
+ goto obd_matches;
+ }
}
if (i == param->lmd->lmd_lmm.lmm_stripe_count)
- decision = -1;
+ goto decided;
}
}
- /* Check the time on mds. */
- if (!decision) {
- int for_mds;
-
- for_mds = lustre_fs ? (S_ISREG(st->st_mode) &&
- param->lmd->lmd_lmm.lmm_stripe_count)
- : 0;
- decision = find_time_check(st, param, for_mds);
- }
+obd_matches:
/* If file still fits the request, ask osd for updated info.
The regulat stat is almost of the same speed as some new
'glimpse-size-ioctl'. */
- if (!decision && param->lmd->lmd_lmm.lmm_stripe_count &&
- S_ISREG(st->st_mode)) {
+ if (!decision && S_ISREG(st->st_mode) &&
+ (param->lmd->lmd_lmm.lmm_stripe_count || param->size_check)) {
if (dir) {
ret = ioctl(dirfd(dir), IOC_LOV_GETINFO,
(void *)param->lmd);
@@ -887,10 +1024,16 @@ static int cb_find_init(char *path, DIR *parent, DIR *dir,
}
/* Check the time on osc. */
- if (!decision)
- decision = find_time_check(st, param, 0);
+ decision = find_time_check(st, param, 0);
+ if (decision == -1)
+ goto decided;
}
+ if (param->size_check)
+ decision = find_value_cmp(st->st_size, param->size,
+ param->size_sign, param->size_units,
+ 0);
+
if (decision != -1) {
printf("%s", path);
if (param->zeroend)
@@ -958,7 +1101,7 @@ static int cb_getstripe(char *path, DIR *parent, DIR *d, void *data,
/* Prepare odb. */
if (!param->got_uuids) {
- ret = setup_obd_uuids(d ? d : parent, path, param);
+ ret = setup_obd_uuid(d ? d : parent, path, param);
if (ret)
return ret;
}
--
GitLab