1 /* rfs_age_unit_base.c */
7 #include "rfs_assert.h"
16 #define MAX_FILES 100000
17 #define MAX_DIRS 100000
18 #define FILE_FH_HTABLE_SIZE MAX_FILES
19 #define MAX_NAMELEN 512
20 #define MAX_PLAY_PATH_SIZE 1024
21 #define MAX_COMMAND_LEN (MAX_PLAY_PATH_SIZE+16)
22 #define NFS_MAXDATA 4096
23 //#define NFS_MAXDATA 32768
24 #define TRACE_FH_SIZE 64
26 #define FH_T_FLAG_FREE 0
27 #define FH_T_FLAG_IN_USE 1
37 #define DISK_FRAGMENT_SIZE 4096
38 //#define FRAGMENT_NUM 5
39 //#define MIN_WRITE_SIZE 512
40 //#define MIN_WRITE_SIZE 2000000000
42 static char ftypename[3][32] = {"FILE", "DIR", "FTYPE_DONT_CARE"};
43 static char activename[3][32] = {"ACTIVE", "INACTIVE", "ACTIVE_DONT_CARE"};
44 static char existname[4][32] = {"EXIST", "NON_EXIST", "EXIST_DONT_CARE", "COMPLETE"};
53 //char trace_fh [TRACE_FH_SIZE+1];
54 char path[MAX_PLAY_PATH_SIZE];
60 //struct generic_entry * htable;
69 profile_t read_line_profile, fgets_profile;
70 char trace_file[MAX_NAMELEN];
71 FILE * profile_fp = NULL;
72 char testdir[MAX_NAMELEN];
74 int active_obj_num = 0;
75 int exist_active_obj_num = 0;
76 static int active_file_num = 0, active_dir_num =0, age_file_num = 0, age_dir_num = 0;
78 int age_create_num = 0;
79 int age_mkdir_num = 0;
80 int assure_create_num = 0;
81 int assure_mkdir_num = 0;
82 int age_write_num = 0;
83 int nonage_write_num = 0;
84 int overlap_write_num = 0;
91 int MIN_WRITE_SIZE = 512;
99 int init_profile_variables()
101 init_profile ("read_line profile", &read_line_profile);
102 init_profile ("fgets profile", &fgets_profile);
105 int init_fh_info (char * name, fh_info_t * fh_infop, int fh_size, int htable_size)
109 RFS_ASSERT (strlen(name) < sizeof(fh_infop->name));
110 strcpy (fh_infop->name, name);
111 fh_infop->fh_max = 0;
112 //fh_infop->index = 0;
113 fh_infop->fh_size = fh_size;
114 //fh_infop->htable_size = htable_size;
115 fh_infop->fh = (fh_t *)malloc (sizeof(fh_t)*fh_size);
116 RFS_ASSERT (fh_infop->fh);
117 //fh_infop->htable = malloc (sizeof(struct*generic_entry)*htable_size);
118 //RFS_ASSERT (fh_infop->htable);
119 printf("initialize %s size %d bytes\n",
120 //name, sizeof(fh_t)*fh_size + sizeof(struct*generic_entry)*htable_size);
121 name, sizeof(fh_t)*fh_size);
123 for (i=0; i<fh_size; i++)
124 fh_infop->fh[i].flag = FH_T_FLAG_FREE;
129 // init_fh_info ("file_fh", &file_fh, MAX_FILES, MAX_FILES);
130 // init_fh_info ("dir_fh", &dir_fh, MAX_DIRS, MAX_DIRS);
131 init_fh_info ("obj_fh", &obj_fh, MAX_FILES+MAX_DIRS, MAX_FILES+MAX_DIRS);
134 int add_fh_t (fh_info_t * fh_table, char * path, int sfh, int psfh, int size, int ftype, int exist_flag, int active_flag)
140 if (active_flag == ACTIVE)
143 RFS_ASSERT (sfh >= fh_table->active_fh_max);
146 printf ("add to %s path %s sfh %d size %d %s %s %s\n", fh_table->name, path, sfh, size,
147 ftypename[ftype], existname[exist_flag], activename[active_flag]);
149 RFS_ASSERT ( (sfh>=0) && (sfh<fh_table->fh_size) );
150 RFS_ASSERT (fh_table->fh[sfh].flag==FH_T_FLAG_FREE);
151 fh_table->fh[sfh].flag = FH_T_FLAG_IN_USE;
152 if (sfh >= fh_table->fh_max)
153 fh_table->fh_max = sfh+1;
154 strcpy (fh_table->fh[sfh].path, path);
155 fh_table->fh[sfh].psfh = psfh;
156 fh_table->fh[sfh].size = size;
157 fh_table->fh[sfh].cur_size = 0;
158 fh_table->fh[sfh].ftype = ftype;
159 fh_table->fh[sfh].exist_flag = exist_flag;
160 if (active_flag == ACTIVE) {
161 if (ftype == IS_FILE)
164 RFS_ASSERT (ftype== IS_DIR);
168 if (ftype == IS_FILE)
171 RFS_ASSERT (ftype== IS_DIR);
175 //print_fh_map(fh_table);
179 int loop_write (int fd, char * buf, int buflen)
185 ret = write (fd, buf+pos, buflen-pos);
188 printf ("fd %d\n", fd);
189 perror ("loop write");
192 if (ret == buflen-pos)
199 int assure_exist(int sfh, char * path, int ftype_flag)
201 char name[MAX_NAMELEN];
204 int non_exist_flag = 0;
209 printf("assure_exist %s\n", path);
211 ret = stat (path, &st);
214 RFS_ASSERT (errno == ENOENT);
223 while (count++<100) {
224 /* copy the next component from path to name */
225 for (; *p!=0 && *p!='/'; p++, q++ )
228 ret = stat (name, &st);
230 RFS_ASSERT (errno == ENOENT)
231 if ((*p)==0 && (ftype_flag==IS_FILE)) {
232 ret = creat (name, S_IRWXU);
235 RFS_ASSERT (ret >=0);
236 assure_create_num ++;
238 printf("sfh %d create %s\n", sfh, name);
241 ret = mkdir (name, S_IRWXU);
245 printf("sfh %d mkdir %s\n", sfh, name);
247 printf("sfh %d middle mkdir %s\n", sfh, name);
249 RFS_ASSERT (ret >=0);
264 int print_fh_map(fh_info_t * fhp)
268 int active_obj_num = 0;
271 for (i=0; i<fhp->fh_max; i++) {
272 if (fhp->fh[i].flag == FH_T_FLAG_IN_USE) {
274 if (i < fhp->active_fh_max)
278 printf("%s[%d] %s %s %s\n", fhp->name, i, fhp->fh[i].path, ftypename[fhp->fh[i].ftype], existname[fhp->fh[i].exist_flag]);
281 fprintf(stderr, "fh_max %d active_fh_max %d, in_use_num %d entries active_obj_num %d \n", fhp->fh_max, fhp->active_fh_max, num, active_obj_num);
284 void read_fh_map(char * fh_map_file)
289 char trace_fh[TRACE_FH_SIZE];
296 char name[MAX_NAMELEN];
299 fp = fopen(fh_map_file, "r");
301 printf ("can not opern %s\n", fh_map_file);
305 RFS_ASSERT (fp!=NULL);
309 memset(buf, 0, sizeof(buf));
310 while (fgets(buf, 1024, fp)) {
311 RFS_ASSERT (fh_map_debug==0);
314 printf ("line %d %s", lineno, buf);
315 if (lineno % 10000==0)
316 printf("%d fh_map entry read\n", lineno);
319 if (!strncmp(buf, "::DIR ", strlen("::DIR "))) {
320 strcpy (name, testdir);
322 sscanf(buf, "::DIR %s %d\n", name+strlen(name), &sfh);
323 add_fh_t (&obj_fh, name, sfh, -1, -1, IS_DIR, NON_EXIST, ACTIVE);
325 RFS_ASSERT (!strncmp(buf,"::DIR Fake 1\n", strlen("::DIR Fake 1\n")));
327 add_fh_t (&obj_fh, name, sfh, -1, -1, IS_DIR, EXIST, ACTIVE);
328 exist_active_obj_num ++;
333 char filename[MAX_NAMELEN];
335 if (!strncmp(buf, "::TBDIR", strlen("::TBDIR")))
338 p = strstr(buf, "parent");
340 sscanf(p, "parent %d\n", &psfh);
341 RFS_ASSERT (obj_fh.fh[psfh].flag == FH_T_FLAG_IN_USE);
342 p = strstr(p, "name");
344 if (!strncmp(p, "name xx", strlen("name xx"))) {
345 sscanf(p, "name xx-%s sfh %d size %x", filename, &sfh, &size);
346 //printf ("name xx-%s sfh %d\n", filename, sfh);
348 sscanf(p, "name \"%s sfh %d size %x", filename, &sfh, &size);
349 //printf ("name %s sfh %d\n", filename, sfh);
350 filename[strlen(filename)-1]=0;
352 strcpy (name, obj_fh.fh[psfh].path);
354 strcat (name, filename);
355 add_fh_t (&obj_fh, name, sfh, psfh, size, IS_FILE, NON_EXIST, ACTIVE);
360 obj_fh.active_fh_max = obj_fh.fh_max;
362 print_fh_map (&obj_fh);
368 printf("agefs ACTIVE_RATIO FRAGMENT_NUM fh_path_map testdir\n");
369 printf("Note: if populate_scale is 4, the total active file size is 1GB\n");
370 printf(" then the total initial file system size is about 4GB\n");
373 inline char * read_line (int disk_index)
375 static FILE * fp=NULL;
377 static int start_disk_index=0;
379 static int finish_flag = 0;
381 #define READ_LINE_BUF_SIZE 1000
382 #define READ_LINE_LENGTH 32
384 static char line_buf[READ_LINE_BUF_SIZE][READ_LINE_LENGTH];
385 start_profile (&read_line_profile);
388 if (strcmp(trace_file, "stdin")) {
389 fp = fopen(trace_file, "r");
391 printf("can not open files %s\n", fp);
397 RFS_ASSERT (fp!=NULL);
398 for (i=0; i<READ_LINE_BUF_SIZE; i++) {
399 start_profile(&fgets_profile);
400 if (!fgets(line_buf[i], READ_LINE_LENGTH, fp)) {
403 end_profile(&fgets_profile);
404 //printf ("read_line, line_buf[%d]:%s", i, line_buf[i]);
408 RFS_ASSERT (disk_index <= start_disk_index+READ_LINE_BUF_SIZE)
409 if (disk_index==(start_disk_index+READ_LINE_BUF_SIZE)) {
413 start_profile(&fgets_profile);
414 if (!fgets(line_buf[start], READ_LINE_LENGTH, fp)) {
415 end_profile(&fgets_profile);
420 end_profile(&fgets_profile);
421 //printf ("read_line, line_buf[%d]:%s", start, line_buf[start]);
422 start = (start+1) % READ_LINE_BUF_SIZE;
425 RFS_ASSERT (disk_index < start_disk_index+READ_LINE_BUF_SIZE)
426 i = (start+disk_index-start_disk_index)%READ_LINE_BUF_SIZE;
428 end_profile (&read_line_profile);
429 return (line_buf[i]);
439 static struct statfs first_stfs;
440 static int first_entry = 1;
442 ret = statfs (testdir, &stfs);
443 RFS_ASSERT (ret == 0);
449 fprintf(stderr, "active_file_num %d active_dir_num %d age_file_num %d age_dir_num %d\n",
450 active_file_num, active_dir_num, age_file_num, age_dir_num);
451 fprintf(stderr, "number of used file nodes %d, used (4K) blocks in fs %d (%d MB)\n", first_stfs.f_ffree-stfs.f_ffree, first_stfs.f_bfree - stfs.f_bfree, (first_stfs.f_bfree-stfs.f_bfree)/(1000000/4096));
452 fprintf(stderr, "assure_create_num %d assure_mkdir_num %d\n", assure_create_num, assure_mkdir_num);
456 int pcnt; /* percentile */
457 int size; /* file size in KB */
458 } sfs_io_file_size_dist;
460 sfs_io_file_size_dist Default_file_size_dist[] = {
461 /* percentage KB size */
475 { 100, 1024}, /* 1% */
480 * For a value between 0-99, return a size based on distribution
485 static file_array_initialized = 0;
486 static int file_size_array[100];
494 if (file_array_initialized == 0) {
496 for (j = 0, k = 0; j < 100; j++) {
497 if (j >= Default_file_size_dist[k].pcnt &&
498 Default_file_size_dist[k + 1].size != 0)
500 file_size_array[j] = Default_file_size_dist[k].size * 1024;
502 file_array_initialized++;
504 return (file_size_array[i]);
507 int range_random(int min, int max)
510 i = random()%(max-min) + min;
514 /* answer 1 with a probability of percent/100 */
515 int decide(int percent)
517 int i = random()%100;
524 int select_obj (fh_info_t * fhp, int ftype, int exist_flag, int active_flag)
530 //printf ("select_obj %s %s %s\n", ftypename[ftype], existname[exist_flag], activename[active_flag]);
531 if (active_flag == ACTIVE) {
532 sfh = range_random (0, fhp->active_fh_max);
533 for (i=0; i<fhp->active_fh_max; i++) {
534 if ((fhp->fh[sfh].flag == FH_T_FLAG_IN_USE) &&
535 ((ftype==DONT_CARE) || (ftype ==fhp->fh[sfh].ftype)) &&
536 ((exist_flag==DONT_CARE) || (fhp->fh[sfh].exist_flag == exist_flag)))
538 sfh = (sfh+1) % fhp->active_fh_max;
541 RFS_ASSERT (active_flag == DONT_CARE);
542 RFS_ASSERT (exist_flag == EXIST);
543 sfh = range_random (0, fhp->fh_max);
544 for (i=0; i<fhp->fh_max; i++) {
545 if ((fhp->fh[sfh].flag == FH_T_FLAG_IN_USE) &&
546 ((ftype==DONT_CARE) || (fhp->fh[sfh].ftype == ftype)) &&
547 (fhp->fh[sfh].exist_flag == EXIST)) {
550 sfh = (sfh+1) % fhp->fh_max;
555 print_fh_map(&obj_fh);
556 printf ("active_obj_num %d exist_active_obj_num %d \n", active_obj_num, exist_active_obj_num);
557 printf ("failed select_obj %s %s\n", ftypename[ftype], activename[active_flag]);
561 /* append "size" to file "path" */
562 int append_file (int sfh, char * path, int size)
565 int written_bytes = 0;
566 static char buf[NFS_MAXDATA];
569 printf ("sfh %d append_file %s size %d\n", sfh, path, size);
571 fd = open (path, O_WRONLY|O_APPEND);
576 while (written_bytes+NFS_MAXDATA < size) {
577 loop_write (fd, buf, NFS_MAXDATA);
578 written_bytes += NFS_MAXDATA;
580 loop_write (fd, buf, size-written_bytes);
584 int get_write_size (int target_size, int cur_size)
587 if (target_size - cur_size < MIN_WRITE_SIZE)
588 return (target_size - cur_size);
590 /* target_size/FRAGMENT_NUM would be the average value of i */
591 if (target_size < FRAGMENT_NUM) {
594 i = random() % (2*(target_size/FRAGMENT_NUM));
595 if (i < MIN_WRITE_SIZE)
598 if (i > (target_size - cur_size))
599 i = target_size - cur_size;
604 int main(int argc, char ** argv)
607 static int disk_index=0;
608 int j, nfs3proc, size, off, count;
613 int ftype_flag = 0, active_flag = 0;
614 char name[MAX_PLAY_PATH_SIZE];
623 ACTIVE_RATIO = atoi(argv[1]);
624 FRAGMENT_NUM = atoi(argv[2]);
626 MIN_WRITE_SIZE = 2000000000;
628 MIN_WRITE_SIZE = DISK_FRAGMENT_SIZE;
630 strcpy (testdir, argv[4]);
631 ret = stat (testdir, &st);
632 if ((ret == -1) && (errno==ENOENT)) {
633 ret = mkdir (testdir, S_IRWXU);
635 RFS_ASSERT (ret >= 0);
636 read_fh_map(argv[3]);
637 print_fh_map(&obj_fh);
638 init_profile_variables();
640 /* NOTE: should have put file and directories in one table */
642 for (i=0; exist_active_obj_num < active_obj_num; i++) {
644 if ((i!=0) && ((i%1000)==0)) {
645 fprintf (stderr, "\n%d object created \n", i);
649 /* decide on the exact active obj or populated obj */
650 if (decide(ACTIVE_RATIO)) {
651 sfh = select_obj (&obj_fh, DONT_CARE, NON_EXIST, ACTIVE);
655 obj_fh.fh[sfh].exist_flag = EXIST;
656 exist_active_obj_num ++;
657 ftype_flag = obj_fh.fh[sfh].ftype;
658 size = obj_fh.fh[sfh].size;
663 while (obj_fh.fh[--tmpfh].exist_flag == NON_EXIST) {
664 if (strstr(obj_fh.fh[sfh].path, obj_fh.fh[tmpfh].path)) {
665 obj_fh.fh[tmpfh].exist_flag = EXIST;
666 //printf ("set %s to exist due to %s\n", obj_fh.fh[tmpfh].path, obj_fh.fh[sfh].path);
667 exist_active_obj_num ++;
673 psfh = select_obj (&obj_fh, IS_DIR, EXIST, DONT_CARE);
674 strcpy (name, obj_fh.fh[psfh].path);
676 sprintf(name+strlen(name), "/AGE%d", obj_fh.fh_max);
678 /* decide next obj is file or directory */
679 if (decide(FILE_RATIO)) {
680 ftype_flag = IS_FILE;
681 size = get_file_size();
686 add_fh_t (&obj_fh, name, sfh, psfh, size, ftype_flag, EXIST, INACTIVE);
690 /* make sure/create the obj pathname on disk */
691 assure_exist (sfh, obj_fh.fh[sfh].path, ftype_flag);
692 /* write file to sizes according certain distribution
693 if (ftype_flag == IS_FILE)
694 append_file (obj_fh.fh[sfh].path, obj_fh.fh[sfh].size);
700 printf ("start writing files\n");
703 sfh = select_obj (&obj_fh, IS_FILE, EXIST, DONT_CARE);
706 write_size = get_write_size (obj_fh.fh[sfh].size, obj_fh.fh[sfh].cur_size);
707 append_file (sfh, obj_fh.fh[sfh].path, write_size);
708 obj_fh.fh[sfh].cur_size += write_size;
709 if (obj_fh.fh[sfh].cur_size == obj_fh.fh[sfh].size) {
710 obj_fh.fh[sfh].exist_flag = COMPLETE;
713 printf ("%d append_file operation performed\n", i);
717 printf ("end of file system object creation\n");
718 print_fh_map(&obj_fh);
723 int create_mkdir_op (int flag)
726 char name[MAX_NAMELEN];
727 char command[MAX_COMMAND_LEN];
731 fh_info_t * fh_infop;
733 while (count++ < 100) {
734 i = random()%dir_fh.fh_max;
735 if (dir_fh.fh[i].flag==FH_T_FLAG_IN_USE) {
736 assure_exist(dir_fh.fh[i].path);
737 strcpy (name, dir_fh.fh[i].path);
738 if (flag == IS_FILE) {
739 sprintf (name+strlen(name), "AGEfile%d", fhno++);
740 fd = creat (name, S_IRWXU);
742 //printf ("create fd %d\n", fd);
746 sprintf (name+strlen(name), "AGEdir%d", fhno++);
747 fd = mkdir (name, S_IRWXU);
753 if (errno == ENOENT) {
754 dir_fh.fh[i].flag = FH_T_FLAG_FREE;
759 add_fh_t (fh_infop, name, EXIST);
773 while (count++<100) {
774 i = random()%file_fh.fh_max;
775 if (file_fh.fh[i].flag == FH_T_FLAG_IN_USE) {
777 if (!strstr(file_fh.fh[i].path, "AGE"))
780 assure_exist(file_fh.fh[i].path);
781 ret = remove (file_fh.fh[i].path);
782 RFS_ASSERT (ret ==0);
783 file_fh.fh[i].flag = FH_T_FLAG_FREE;
794 char command[MAX_COMMAND_LEN];
797 while (count++<100) {
798 i = random()%dir_fh.fh_max;
799 if ( (dir_fh.fh[i].flag == FH_T_FLAG_IN_USE) ) {
801 if (!strstr(file_fh.fh[i].path, "AGE"))
804 assure_exist(file_fh.fh[i].path);
805 ret = rmdir (dir_fh.fh[i].path);
807 dir_fh.fh[i].flag = FH_T_FLAG_FREE;
810 RFS_ASSERT ((ret == -1) && (errno == ENOTEMPTY));
811 //strcpy (command, "rm -r %s", dir_fh.fh[i].path);
818 int write_op (int off, int size)
820 static char buf[NFS_MAXDATA];
827 RFS_ASSERT (size <= NFS_MAXDATA);
828 while (count++<100) {
829 i = random()%file_fh.fh_max;
830 if ( (file_fh.fh[i].flag == FH_T_FLAG_IN_USE) ) {
831 assure_exist(file_fh.fh[i].path);
832 fd = open(file_fh.fh[i].path, O_WRONLY);
836 //printf ("write fd %d\n", fd);
839 if (st.st_size < (off+size)) {
840 int written_bytes = 0;
841 while (written_bytes+NFS_MAXDATA < off+size-st.st_size) {
842 loop_write (fd, buf, NFS_MAXDATA);
843 written_bytes += NFS_MAXDATA;
845 loop_write (fd, buf, off+size-st.st_size-written_bytes);
846 if (strstr(file_fh.fh[i].path, "AGE")) {
847 age_write_num+=(written_bytes+NFS_MAXDATA-1)/NFS_MAXDATA;
849 nonage_write_num+=(written_bytes+NFS_MAXDATA-1)/NFS_MAXDATA;
853 if (strstr(file_fh.fh[i].path, "AGE")) {
857 loop_write (fd, buf, size);
866 int truncate_op(int size)
872 while (count++<100) {
873 i = random()%file_fh.fh_max;
874 if ( (file_fh.fh[i].flag == FH_T_FLAG_IN_USE) ) {
876 if (!strstr(file_fh.fh[i].path, "AGE"))
879 assure_exist (file_fh.fh[i].path);
880 ret = truncate(file_fh.fh[i].path, size);
883 RFS_ASSERT (errno == ENOENT);
884 file_fh.fh[i].flag = FH_T_FLAG_FREE;
893 char name[MAX_NAMELEN];
894 int sfh; /* file to be aged */
900 /* get the sfh and size of the selected file to be aged */
901 sfh = select_obj (&obj_fh, IS_FILE, EXIST, ACTIVE);
902 ret = stat (obj_fh.fh[sfh].path, &st);
903 RFS_ASSERT (ret == 0);
904 ret = truncate(obj_fh.fh[i].path, st.st_size/2);
907 psfh = obj_fh.fh[sfh].psfh;
908 strcpy (name, obj_fh.fh[psfh].path);
909 sprintf(name+strlen(name), "/AGE%d", obj_fh.fh_max);
910 agefd = creat (name, S_IRWXU);
911 //write (agefs, buf, 0);
913 add_fh_t (&obj_fh, name, sfh, psfh, size, ftype_flag, EXIST, INACTIVE);