1 /* rfs_age_unit_base.c */
7 #include "rfs_assert.h"
16 #define MAX_FILES 400000
17 #define MAX_DIRS 100000
18 #define FILE_FH_HTABLE_SIZE MAX_FILES
19 #define MAX_NAMELEN 512
20 #define MAX_PLAY_PATH_SIZE 256
21 //#define MAX_COMMAND_LEN (MAX_PLAY_PATH_SIZE+16)
22 //#define NFS_MAXDATA 4096
23 #define NFS_MAXDATA 32768
24 #define TRACE_FH_SIZE 64
26 #define FH_T_FLAG_FREE 0
27 #define FH_T_FLAG_IN_USE 1
37 int EVEN_CHUNK_SIZE = 0;
40 static char ftypename[3][32] = {"FILE", "DIR", "FTYPE_DONT_CARE"};
41 static char activename[3][32] = {"ACTIVE", "INACTIVE", "ACTIVE_DONT_CARE"};
42 static char existname[4][32] = {"EXIST", "NON_EXIST", "EXIST_DONT_CARE", "COMPLETE"};
51 int accumulated_write_size;
52 //char trace_fh [TRACE_FH_SIZE+1];
53 char path[MAX_PLAY_PATH_SIZE];
59 //struct generic_entry * htable;
68 profile_t read_line_profile, fgets_profile;
69 char trace_file[MAX_NAMELEN];
70 FILE * profile_fp = NULL;
71 char testdir[MAX_NAMELEN];
73 int active_obj_num = 0;
74 int exist_active_obj_num = 0;
75 static int active_file_num = 0, active_dir_num =0, age_file_num = 0, age_dir_num = 0;
77 int age_create_num = 0;
78 int age_mkdir_num = 0;
79 int assure_create_num = 0;
80 int assure_mkdir_num = 0;
81 int age_write_num = 0;
82 int nonage_write_num = 0;
83 int overlap_write_num = 0;
85 int fs_size_MB = 0, fs_size = 0;
92 int MAX_FS_SIZE_MB = 1000000;
93 int DISK_FRAGMENT_SIZE = 4096;
94 int DISK_FRAGMENT_SIZE_MASK = 0xFFFFF000;
106 int init_profile_variables()
108 init_profile ("read_line profile", &read_line_profile);
109 init_profile ("fgets profile", &fgets_profile);
112 int init_fh_info (char * name, fh_info_t * fh_infop, int fh_size, int htable_size)
116 RFS_ASSERT (strlen(name) < sizeof(fh_infop->name));
117 strcpy (fh_infop->name, name);
118 fh_infop->fh_max = 0;
119 //fh_infop->index = 0;
120 fh_infop->fh_size = fh_size;
121 //fh_infop->htable_size = htable_size;
122 fh_infop->fh = (fh_t *)malloc (sizeof(fh_t)*fh_size);
123 RFS_ASSERT (fh_infop->fh);
124 //fh_infop->htable = malloc (sizeof(struct*generic_entry)*htable_size);
125 //RFS_ASSERT (fh_infop->htable);
126 printf("initialize %s size %d bytes\n",
127 //name, sizeof(fh_t)*fh_size + sizeof(struct*generic_entry)*htable_size);
128 name, sizeof(fh_t)*fh_size);
130 for (i=0; i<fh_size; i++)
131 fh_infop->fh[i].flag = FH_T_FLAG_FREE;
136 // init_fh_info ("file_fh", &file_fh, MAX_FILES, MAX_FILES);
137 // init_fh_info ("dir_fh", &dir_fh, MAX_DIRS, MAX_DIRS);
138 init_fh_info ("obj_fh", &obj_fh, MAX_FILES+MAX_DIRS, MAX_FILES+MAX_DIRS);
141 int add_fh_t (fh_info_t * fh_table, char * path, int sfh, int psfh, int size, int ftype, int exist_flag, int active_flag)
146 fs_size += DISK_FRAGMENT_SIZE;
148 fs_size += ((size+DISK_FRAGMENT_SIZE-1) & DISK_FRAGMENT_SIZE_MASK);
149 if (size > (DISK_FRAGMENT_SIZE*12)) // first indirect block
150 fs_size += DISK_FRAGMENT_SIZE;
152 if (fs_size > 1000000) {
153 fs_size_MB += fs_size/1000000;
154 fs_size = fs_size % 1000000;
159 if (active_flag == ACTIVE)
162 RFS_ASSERT (sfh >= fh_table->active_fh_max);
165 printf ("add to %s path %s sfh %d size %d %s %s %s\n", fh_table->name, path, sfh, size,
166 ftypename[ftype], existname[exist_flag], activename[active_flag]);
168 RFS_ASSERT ( (sfh>=0) && (sfh<fh_table->fh_size) );
169 RFS_ASSERT (fh_table->fh[sfh].flag==FH_T_FLAG_FREE);
170 fh_table->fh[sfh].flag = FH_T_FLAG_IN_USE;
171 if (sfh >= fh_table->fh_max)
172 fh_table->fh_max = sfh+1;
173 RFS_ASSERT (strlen(path)<MAX_PLAY_PATH_SIZE);
174 strcpy (fh_table->fh[sfh].path, path);
175 fh_table->fh[sfh].psfh = psfh;
176 fh_table->fh[sfh].size = size;
177 fh_table->fh[sfh].cur_size = 0;
178 fh_table->fh[sfh].accumulated_write_size = 0;
179 fh_table->fh[sfh].ftype = ftype;
180 fh_table->fh[sfh].exist_flag = exist_flag;
181 if (active_flag == ACTIVE) {
182 if (ftype == IS_FILE)
185 RFS_ASSERT (ftype== IS_DIR);
189 if (ftype == IS_FILE)
192 RFS_ASSERT (ftype== IS_DIR);
196 //print_fh_map(fh_table);
200 int loop_write (int fd, char * buf, int buflen)
206 ret = write (fd, buf+pos, buflen-pos);
209 RFS_ASSERT (errno == ENOSPC);
211 perror ("loop write");
212 check_free_blocks(0);
215 if (ret == buflen-pos)
222 int assure_exist(int sfh, char * path, int ftype_flag)
224 char name[MAX_NAMELEN];
227 int non_exist_flag = 0;
232 printf("assure_exist %s\n", path);
234 ret = stat (path, &st);
236 if (ftype_flag == IS_DIR) {
237 RFS_ASSERT (st.st_mode & S_IFDIR);
238 RFS_ASSERT (!(st.st_mode & S_IFREG));
240 RFS_ASSERT (st.st_mode & S_IFREG);
241 RFS_ASSERT (!(st.st_mode & S_IFDIR));
248 //RFS_ASSERT (errno == ENOENT);
257 while (count++<100) {
258 /* copy the next component from path to name */
259 for (; *p!=0 && *p!='/'; p++, q++ )
262 ret = stat (name, &st);
266 RFS_ASSERT (errno == ENOENT)
267 if ((*p)==0 && (ftype_flag==IS_FILE)) {
268 ret = creat (name, S_IRWXU);
271 RFS_ASSERT (ret >=0);
272 assure_create_num ++;
274 printf("sfh %d create %s\n", sfh, name);
277 ret = mkdir (name, S_IRWXU);
280 RFS_ASSERT (ret >=0);
284 printf("sfh %d mkdir %s\n", sfh, name);
286 printf("sfh %d middle mkdir %s\n", sfh, name);
288 RFS_ASSERT (ret ==0);
303 int print_fh_map(fh_info_t * fhp)
307 int active_obj_num = 0;
310 for (i=0; i<fhp->fh_max; i++) {
311 if (fhp->fh[i].flag == FH_T_FLAG_IN_USE) {
313 if (i < fhp->active_fh_max)
317 printf("%s[%d] %s %s %s\n", fhp->name, i, fhp->fh[i].path, ftypename[fhp->fh[i].ftype], existname[fhp->fh[i].exist_flag]);
320 fprintf(stderr, "fh_max %d active_fh_max %d, in_use_num %d entries active_obj_num %d \n", fhp->fh_max, fhp->active_fh_max, num, active_obj_num);
323 void read_fh_map_line_skimmer (char * buf)
326 char name[MAX_NAMELEN];
328 char filename[MAX_NAMELEN];
331 if (!strncmp(buf, "::DIR ", strlen("::DIR "))) {
332 strcpy (name, testdir);
334 sscanf(buf, "::DIR %s %d\n", name+strlen(name), &sfh);
335 add_fh_t (&obj_fh, name, sfh, -1, -1, IS_DIR, NON_EXIST, ACTIVE);
337 RFS_ASSERT (!strncmp(buf,"::DIR Fake 1\n", strlen("::DIR Fake 1\n")));
339 add_fh_t (&obj_fh, name, sfh, -1, -1, IS_DIR, EXIST, ACTIVE);
340 exist_active_obj_num ++;
344 if (!strncmp(buf, "::TBDIR", strlen("::TBDIR")))
347 p = strstr(buf, "parent");
349 sscanf(p, "parent %d\n", &psfh);
350 RFS_ASSERT (obj_fh.fh[psfh].flag == FH_T_FLAG_IN_USE);
351 p = strstr(p, "name");
353 if (!strncmp(p, "name xx", strlen("name xx"))) {
354 sscanf(p, "name xx-%s sfh %d size %x", filename, &sfh, &size);
355 //printf ("name xx-%s sfh %d\n", filename, sfh);
357 sscanf(p, "name \"%s sfh %d size %x", filename, &sfh, &size);
358 //printf ("name %s sfh %d\n", filename, sfh);
359 filename[strlen(filename)-1]=0;
361 strcpy (name, obj_fh.fh[psfh].path);
363 strcat (name, filename);
364 add_fh_t (&obj_fh, name, sfh, psfh, size, IS_FILE, NON_EXIST, ACTIVE);
368 void read_fh_map_line_ls (char * buf)
372 char name[MAX_NAMELEN];
375 strcpy (name, testdir);
377 if (strchr(buf, ' ')) {
378 sscanf(buf, "%d %s\n", &size, p+strlen(name));
379 add_fh_t (&obj_fh, name, sfh, -1, size, IS_FILE, NON_EXIST, ACTIVE);
381 sscanf(buf, "%s\n", p+strlen(name));
382 add_fh_t (&obj_fh, name, sfh, -1, -1, IS_DIR, NON_EXIST, ACTIVE);
387 void read_fh_map (char * fh_map_file)
394 #define FH_MAP_FORMAT_SKIMMER 0
395 #define FH_MAP_FORMAT_LS 1
398 if (strstr(fh_map_file, ".ski"))
399 fh_map_format = FH_MAP_FORMAT_SKIMMER;
401 fh_map_format = FH_MAP_FORMAT_LS;
403 fp = fopen(fh_map_file, "r");
405 printf ("can not opern %s\n", fh_map_file);
409 RFS_ASSERT (fp!=NULL);
412 memset(buf, 0, sizeof(buf));
414 while (fgets(buf, 1024, fp)) {
415 RFS_ASSERT (fh_map_debug==0);
418 printf ("line %d %s", lineno, buf);
419 if (lineno % 10000==0)
420 printf("%d fh_map entry read\n", lineno);
421 if (fh_map_format == FH_MAP_FORMAT_SKIMMER)
422 read_fh_map_line_skimmer(buf);
424 read_fh_map_line_ls (buf);
428 obj_fh.active_fh_max = obj_fh.fh_max;
430 print_fh_map (&obj_fh);
436 printf("\n\nagefs fmt4 HOLDER_NUM HOLDER_SIZE DISK_FRAGMENT_SIZE testdir\n");
437 printf("Note: fmt4 is used to initialize the holders in a logical partition before starting writing aged files in a specific pattern as by fmt3 command\n");
439 printf("\n\nagefs fmt3 aged_file CHUNK_SIZE CHUNK_DISTANCE CHUNK_NUM START_BNO HOLDER_SIZE HOLDER_NUM DISK_FRAGMENT_SIZE testdir\n");
440 printf("Note: one file is written as CHUNK_NUM number of continuous chunks on disk, each chunk is of CHUNK_SIZE blocks, the distance between two adjacent chunks is CHUNK_DISTANCE blocks\n");
442 printf("\n\nagefs fmt2 size1 .. size_n num testdir\n");
443 printf("Note: N file is writen interleavingly for _num_ times, each time _size1_ bytes is written to file1, _size2_ bytes is written to file2, _sizen_ bytes is written to filen\n");
445 printf("\n\nagefs EVEN_CHUNK_SIZE FILE_RATIO ACTIVE_RATIO WRITE_CHUNK_NUM MAX_FS_SIZE_MB STAGE_NUM fh_path_map testdir\n");
446 printf("Note: EVEN_CHUNK_SIZE: if 1, each file is chopped to equal size chunks, if 0, each file size is chopped randomly but with the average size to be CHUNK_SIZE");
447 printf(" FILE_RATIO: percentage of number of inactive files over number of inactive file system objects\n");
448 printf(" ACTIVE_RATIO: percentage of number of active file system objects over all file system objects\n");
449 printf(" WRITE_CHUNK_NUM: when a file is initialized, it is written in several open-close session interleved with writes to other files. Except small files where file_size/WRITE_CHUNK_SIZE is less than DISK_FRAGMENT_SIZE, each open-close session on the average write (file_size/WRITE_CHUNK_NUM) bytes. \n");
450 printf(" MAX_FS_SIZE_MB: another condition to stop initialization, either all active file is initialized, or max file system size is reached\n");
451 printf(" STAGE_NUM: divide the writing of files into several stages, each stage finish initialization of some of the files. The bigger the STAGE_NUM, the less concurrency is there\n");
454 inline char * read_line (int disk_index)
456 static FILE * fp=NULL;
458 static int start_disk_index=0;
460 static int finish_flag = 0;
462 #define READ_LINE_BUF_SIZE 1000
463 #define READ_LINE_LENGTH 32
465 static char line_buf[READ_LINE_BUF_SIZE][READ_LINE_LENGTH];
466 start_profile (&read_line_profile);
469 if (strcmp(trace_file, "stdin")) {
470 fp = fopen(trace_file, "r");
472 printf("can not open files %s\n", fp);
478 RFS_ASSERT (fp!=NULL);
479 for (i=0; i<READ_LINE_BUF_SIZE; i++) {
480 start_profile(&fgets_profile);
481 if (!fgets(line_buf[i], READ_LINE_LENGTH, fp)) {
484 end_profile(&fgets_profile);
485 //printf ("read_line, line_buf[%d]:%s", i, line_buf[i]);
489 RFS_ASSERT (disk_index <= start_disk_index+READ_LINE_BUF_SIZE)
490 if (disk_index==(start_disk_index+READ_LINE_BUF_SIZE)) {
494 start_profile(&fgets_profile);
495 if (!fgets(line_buf[start], READ_LINE_LENGTH, fp)) {
496 end_profile(&fgets_profile);
501 end_profile(&fgets_profile);
502 //printf ("read_line, line_buf[%d]:%s", start, line_buf[start]);
503 start = (start+1) % READ_LINE_BUF_SIZE;
506 RFS_ASSERT (disk_index < start_disk_index+READ_LINE_BUF_SIZE)
507 i = (start+disk_index-start_disk_index)%READ_LINE_BUF_SIZE;
509 end_profile (&read_line_profile);
510 return (line_buf[i]);
517 static struct statfs first_stfs;
518 static int first_entry = 1;
520 ret = statfs (testdir, &stfs);
521 RFS_ASSERT (ret == 0);
527 fprintf(stderr, "active_file_num %d active_dir_num %d age_file_num %d age_dir_num %d\n",
528 active_file_num, active_dir_num, age_file_num, age_dir_num);
529 fprintf(stderr, "number of used file nodes %d, used (4K) blocks in fs %d (%d MB)\n", first_stfs.f_ffree-stfs.f_ffree, first_stfs.f_bfree - stfs.f_bfree, (first_stfs.f_bfree-stfs.f_bfree)/(1000000/DISK_FRAGMENT_SIZE));
530 fprintf(stderr, "assure_create_num %d assure_mkdir_num %d\n", assure_create_num, assure_mkdir_num);
534 int pcnt; /* percentile */
535 int size; /* file size in KB */
536 } sfs_io_file_size_dist;
538 sfs_io_file_size_dist Default_file_size_dist[] = {
539 /* percentage KB size */
554 { 100, 1024}, /* 1% */
559 * For a value between 0-99, return a size based on distribution
564 static file_array_initialized = 0;
565 static int file_size_array[100];
573 if (file_array_initialized == 0) {
575 for (j = 0, k = 0; j < 100; j++) {
576 if (j >= Default_file_size_dist[k].pcnt &&
577 Default_file_size_dist[k + 1].size != 0)
579 file_size_array[j] = Default_file_size_dist[k].size * 1024;
581 file_array_initialized++;
583 return (file_size_array[i]);
586 int range_random(int min, int max)
589 i = (random()%(max-min)) + min;
593 /* answer 1 with a probability of percent/100 */
594 int decide(int percent)
596 int i = random()%100;
603 int select_obj (fh_info_t * fhp, int ftype, int exist_flag, int active_flag, int min, int max)
608 //printf ("select_obj %s %s %s\n", ftypename[ftype], existname[exist_flag], activename[active_flag]);
609 if (active_flag == ACTIVE) {
610 sfh = range_random (0, fhp->active_fh_max);
611 for (i=0; i<fhp->active_fh_max; i++) {
612 if ((fhp->fh[sfh].flag == FH_T_FLAG_IN_USE) &&
613 ((ftype==DONT_CARE) || (ftype ==fhp->fh[sfh].ftype)) &&
614 ((exist_flag==DONT_CARE) || (fhp->fh[sfh].exist_flag == exist_flag))) {
617 sfh = (sfh+1) % fhp->active_fh_max;
622 //printf ("select_obj min %d max %d\n", min, max);
623 RFS_ASSERT (active_flag == DONT_CARE);
624 RFS_ASSERT (exist_flag == EXIST);
625 sfh = range_random (min, max);
626 for (i=min; i<max; i++) {
627 // printf("check %d\n", sfh);
628 RFS_ASSERT ((sfh>=min) && (sfh<max));
629 if ((fhp->fh[sfh].flag == FH_T_FLAG_IN_USE) &&
630 ((ftype==DONT_CARE) || (fhp->fh[sfh].ftype == ftype)) &&
631 (fhp->fh[sfh].exist_flag == EXIST)) {
640 for (i=min; i<max; i++) {
641 if ((fhp->fh[i].flag == FH_T_FLAG_IN_USE) &&
642 (fhp->fh[i].ftype == IS_FILE) ) {
643 if (fhp->fh[i].exist_flag == EXIST) {
644 printf ("actually %d\n", i);
646 RFS_ASSERT (fhp->fh[i].exist_flag != EXIST);
647 RFS_ASSERT (fhp->fh[i].exist_flag == COMPLETE);
654 /* append "size" to file "path" */
655 int append_file (int sfh, char * path, int size)
658 int written_bytes = 0;
660 #define BUF_SIZE 32768
661 static char buf[BUF_SIZE];
664 printf ("sfh %d append_file %s size %d\n", sfh, path, size);
666 fd = open (path, O_WRONLY|O_APPEND);
671 while (written_bytes+NFS_MAXDATA < size) {
672 ret = loop_write (fd, buf, NFS_MAXDATA);
673 RFS_ASSERT (ret!=-1);
674 written_bytes += NFS_MAXDATA;
676 ret = loop_write (fd, buf, size-written_bytes);
677 RFS_ASSERT (ret!=-1);
681 int get_write_size (int target_size, int cur_size)
685 if (target_size - cur_size < MIN_WRITE_SIZE)
686 return (target_size - cur_size);
688 /* target_size/WRITE_CHUNK_NUM would be the average value of i */
689 if (target_size < WRITE_CHUNK_NUM) {
694 i = target_size/WRITE_CHUNK_NUM;
696 i = random() % (2*(target_size/WRITE_CHUNK_NUM));
698 if (i < MIN_WRITE_SIZE)
702 if (i > (target_size - cur_size))
703 i = target_size - cur_size;
720 #define MAX_DISK_FRAGMENT_SIZE 4096
721 #define MAX_HOLDER_SIZE 10
722 #define HOLDER_DIR_NUM 1
723 #define DUMMY_FILE_COUNT 1000
725 main4(int argc, char ** argv)
739 HOLDER_NUM = atoi(argv[i++]);
740 HOLDER_SIZE = atoi(argv[i++]);
741 DISK_FRAGMENT_SIZE = atoi(argv[i++]);
742 RFS_ASSERT (DISK_FRAGMENT_SIZE <= MAX_DISK_FRAGMENT_SIZE);
743 DISK_FRAGMENT_SIZE_MASK = ~(DISK_FRAGMENT_SIZE-1);
744 RFS_ASSERT ((DISK_FRAGMENT_SIZE_MASK == 0xFFFFF000) ||
745 (DISK_FRAGMENT_SIZE_MASK == 0xFFFFFC00) );
746 strcpy (testdir, argv[i]);
748 fprintf(fplog, "main4: initialize the holders HOLDER_NUM %d HOLDER_SIZE %d DISK_FRAGMENT_SIZE %d testdir %s\n",
749 HOLDER_NUM, HOLDER_SIZE, DISK_FRAGMENT_SIZE, testdir);
752 buf = (char *)malloc (HOLDER_SIZE*DISK_FRAGMENT_SIZE);
755 /* create some dummy files */
756 for (i=0; i<DUMMY_FILE_COUNT; i++) {
757 memset (name, 0, sizeof(name));
758 sprintf(name, "%s/dummy%d", testdir, i);
759 fd = creat (name, S_IRWXU);
767 /* create directories */
768 if (HOLDER_DIR_NUM !=1) {
769 for (i=0; i<HOLDER_DIR_NUM; i++) {
770 memset (name, 0, sizeof(name));
771 sprintf(name, "%s/%d", testdir, i);
772 ret = mkdir (name, S_IRWXU);
780 /* create regular files */
781 for (j=0; j<HOLDER_DIR_NUM; j++) {
782 for (i=0; i<HOLDER_NUM/HOLDER_DIR_NUM; i++) {
783 if (HOLDER_DIR_NUM == 1)
784 sprintf(name, "%s/%d", testdir, i);
786 sprintf(name, "%s/%d/%d", testdir, j, i);
787 fd = open (name, O_CREAT|O_WRONLY);
792 ret = loop_write (fd, buf, HOLDER_SIZE*DISK_FRAGMENT_SIZE);
800 /* delete the dummy files */
801 for (i=0; i<DUMMY_FILE_COUNT; i++) {
802 memset (name, 0, sizeof(name));
803 sprintf(name, "%s/dummy%d", testdir, i);
813 int append_space_occupier()
815 static char name [1024];
816 static FILE * fp = NULL;
817 char buf[MAX_DISK_FRAGMENT_SIZE];
820 sprintf(name, "%s/space_ocuupier", testdir);
821 fp = fopen (name, "a+");
822 RFS_ASSERT (fp!= NULL);
823 ret = fwrite (buf, DISK_FRAGMENT_SIZE, 1, fp);
825 perror("append space occupier");
830 ret = fwrite (buf, DISK_FRAGMENT_SIZE, 1, fp);
832 perror("append space occupier");
834 RFS_ASSERT (ret == 1);
837 int create_one_dummy_file()
840 static int index = 0;
841 static char name[1024];
844 for (i=0; i<DUMMY_FILE_COUNT; i++) {
845 index = (index+1) % DUMMY_FILE_COUNT;
846 sprintf(name, "%s/dummy%d", testdir, i);
847 ret = stat (name, &st);
849 RFS_ASSERT (errno == ENOENT);
850 fd = open (name, O_CREAT|O_WRONLY);
858 int delete_one_dummy_file()
861 static int index = 0;
862 static char name[1024];
865 for (i=0; i<DUMMY_FILE_COUNT; i++) {
866 index = (index+1) % DUMMY_FILE_COUNT;
867 sprintf(name, "%s/dummy%d", testdir, i);
868 ret = stat (name, &st);
871 RFS_ASSERT (ret == 0);
874 RFS_ASSERT (errno == ENOENT);
879 int create_sub_holder_file(int holderno, int sub_holderno)
883 static char buf[MAX_DISK_FRAGMENT_SIZE];
886 RFS_ASSERT (MAX_DISK_FRAGMENT_SIZE >= DISK_FRAGMENT_SIZE);
888 sprintf(name, "%d.%d", holderno, sub_holderno);
889 printf ("create/write %s\n", name);
890 fd = open (name, O_CREAT|O_WRONLY);
892 loop_write (fd, buf, DISK_FRAGMENT_SIZE);
896 int get_free_blocks ()
898 static struct statfs stfs;
901 ret = statfs (testdir, &stfs);
902 RFS_ASSERT (ret == 0);
903 return (stfs.f_bfree);
906 int print_free_blocks (char *string)
908 static struct statfs stfs;
911 ret = statfs (testdir, &stfs);
912 RFS_ASSERT (ret == 0);
913 printf("%s f_bfree %d \n", string, stfs.f_bfree);
916 int check_free_blocks (int num)
918 static struct statfs stfs;
921 ret = statfs (testdir, &stfs);
922 RFS_ASSERT (ret == 0);
923 if (stfs.f_bfree!=num) {
924 printf("f_bfree %d expected %d\n", stfs.f_bfree, num);
929 int progress_on_aged_file(int * num)
931 char buf[MAX_DISK_FRAGMENT_SIZE*MAX_HOLDER_SIZE];
932 static need_indirect_blocks = 0;
933 static skip_for_indirect_blocks = 0;
934 static int blkno = 0;
937 print_free_blocks("progress_on_aged_file begin");
939 if (skip_for_indirect_blocks == need_indirect_blocks) {
940 //check_free_blocks(free_block_num);
941 //RFS_ASSERT (free_block_num >= (1+need_indirect_blocks));
943 printf("append to aged file %d bytes\n", DISK_FRAGMENT_SIZE);
944 append_file (0, agename, DISK_FRAGMENT_SIZE);
945 //*free_block_num -= (need_indirect_blocks +1)
946 //check_free_blocks(free_block_num);
949 if (((blkno - 12) % INDIRECT_FANOUT) == 0) {
950 if (((blkno - (INDIRECT_FANOUT+12)) % (INDIRECT_FANOUT*INDIRECT_FANOUT)) == 0) {
951 if (blkno == 12 + INDIRECT_FANOUT + INDIRECT_FANOUT*INDIRECT_FANOUT) {
952 printf ("need_indirect_blocks is set to 3 blkno %d\n", blkno);
953 need_indirect_blocks = 3;
955 printf ("need_indirect_blocks is set to 2 blkno %d\n", blkno);
956 need_indirect_blocks = 2;
959 printf ("need_indirect_blocks is set to 1 blkno %d\n", blkno);
960 need_indirect_blocks = 1;
963 need_indirect_blocks = 0;
965 skip_for_indirect_blocks = 0;
967 skip_for_indirect_blocks ++;
970 printf ("skip_for_indirect_blocks -- %d\n", skip_for_indirect_blocks);
971 print_free_blocks("progress_on_aged_file end");
974 int free_blocks (char * agename, int start, int num)
983 printf ("free_blocks start %d num %d\n", start, num);
986 check_free_blocks(0);
989 holderno = start/HOLDER_SIZE;
990 sub_holderno = start%HOLDER_SIZE;
992 sprintf (name, "%d", holderno);
994 ret = stat (name, &st);
996 RFS_ASSERT (errno == ENOENT);
997 for (i=sub_holderno; (i<HOLDER_SIZE && num>0); i++) {
998 sprintf(name, "%d.%d", holderno, i);
999 ret = stat (name, &st);
1002 printf ("sub_holder file %s is unlinked\n", name);
1004 RFS_ASSERT (ret == 0);
1006 create_one_dummy_file();
1008 printf ("write to age file %d bytes\n", DISK_FRAGMENT_SIZE);
1010 progress_on_aged_file(&num);
1013 printf ("sub_holder file %s is already used\n", name);
1014 RFS_ASSERT ((ret == -1) && (errno ==ENOENT));
1021 RFS_ASSERT (ret == 0);
1022 RFS_ASSERT (st.st_size == HOLDER_SIZE * DISK_FRAGMENT_SIZE);
1023 printf ("holder file %s is unlinked\n", name);
1025 RFS_ASSERT (ret == 0);
1026 check_free_blocks(HOLDER_SIZE);
1028 /* create the sub holders before the first slot that we need */
1029 for (i=0; i<sub_holderno; i++) {
1030 delete_one_dummy_file();
1031 create_sub_holder_file (holderno, i);
1035 i = (HOLDER_SIZE - sub_holderno) < num? (HOLDER_SIZE - sub_holderno): num;
1039 check_free_blocks (i);
1040 ret = loop_write (agefd, buf, DISK_FRAGMENT_SIZE*i);
1041 RFS_ASSERT (ret != -1);
1043 i = HOLDER_SIZE - sub_holderno;
1044 check_free_blocks(i);
1046 while ((sub_holderno < HOLDER_SIZE) && (num>0)) {
1050 progress_on_aged_file(&num);
1052 RFS_ASSERT (ret != -1);
1055 /* create the sub holders after the slot that we need */
1056 for (i=sub_holderno; i<HOLDER_SIZE; i++) {
1057 delete_one_dummy_file();
1058 create_sub_holder_file (holderno, i);
1061 create_one_dummy_file();
1065 int main3(int argc, char ** argv)
1082 CHUNK_SIZE = atoi(argv[i++]);
1083 CHUNK_DISTANCE = atoi (argv[i++]);
1084 CHUNK_NUM = atoi (argv[i++]);
1085 START_BNO = atoi (argv[i++]);
1086 HOLDER_SIZE = atoi (argv[i++]);
1087 RFS_ASSERT (HOLDER_SIZE <= MAX_HOLDER_SIZE);
1088 HOLDER_NUM = atoi (argv[i++]);
1089 DISK_FRAGMENT_SIZE = atoi (argv[i++]);
1090 RFS_ASSERT (DISK_FRAGMENT_SIZE <= MAX_DISK_FRAGMENT_SIZE);
1091 INDIRECT_FANOUT = (DISK_FRAGMENT_SIZE/sizeof(int));
1092 strcpy (testdir, argv[i++]);
1093 strcpy (agename, testdir);
1094 strcat (agename, argv[i++]);
1095 ret = stat (agename, &st);
1097 printf ("%s already exists\n", agename);
1099 RFS_ASSERT (errno == ENOENT);
1101 fprintf(fplog, "main3: to age one file %s in a customized way, CHUNK_SIZE %d CHUNK_DISTANCE %d CHUNK_NUM %d START_BNO %d HOLDER_SIZE %d HOLDER_NUM %d DISK_FRAGMENT_SIZE %d MAX_DISK_FRAGMENT_SIZE %d testdir %s\n", agename, CHUNK_SIZE, CHUNK_DISTANCE, CHUNK_NUM, START_BNO, HOLDER_SIZE, HOLDER_NUM, DISK_FRAGMENT_SIZE, MAX_DISK_FRAGMENT_SIZE, testdir);
1104 /* change working directory */
1105 ret = getcwd(cwd, sizeof(cwd));
1106 RFS_ASSERT (ret == cwd);
1107 i = chdir (testdir);
1110 if (START_BNO == -1) {
1111 block_anchor = random() % (HOLDER_NUM*HOLDER_SIZE);
1113 RFS_ASSERT (START_BNO >=0);
1114 block_anchor = START_BNO % (HOLDER_NUM*HOLDER_SIZE);
1116 while (get_free_blocks()!=0) {
1117 print_free_blocks("fill up initial file system blocks using space occupier");
1118 append_file (0, "/b6/space_occupier", DISK_FRAGMENT_SIZE);
1120 delete_one_dummy_file();
1121 agefd = open (agename, O_CREAT|O_WRONLY);
1122 RFS_ASSERT (agefd>=0);
1125 buf = (char *)malloc (CHUNK_SIZE*DISK_FRAGMENT_SIZE);
1128 for (i=0; i<CHUNK_NUM; i++) {
1129 block_anchor = (block_anchor + CHUNK_DISTANCE) % (HOLDER_NUM * HOLDER_SIZE);
1130 block_anchor = free_blocks (agename, block_anchor, CHUNK_SIZE);
1133 check_free_blocks(0);
1138 int main2(int argc, char ** argv)
1150 num = atoi(argv[argc-2]);
1151 strcpy (testdir, argv[argc-1]);
1152 fprintf(fplog, "main2: generate interleaved files\n");
1153 fprintf(fplog, "testdir %s number of files %d ", testdir, num);
1154 for (i=0; i<argc-4; i++) {
1155 size[i] = atoi(argv[i+2]);
1156 fprintf(fplog, "size[%d] %d ", i, size[i]);
1158 RFS_ASSERT (size[i] >=0 && size[i] < 1000000000);
1159 strcpy (name[i], testdir);
1160 sprintf (name[i], "%s/file%d", testdir, i);
1161 sprintf(cmd, "touch %s", name[i]);
1163 printf ("write %s \n", name[i]);
1165 fprintf(fplog, "\n");
1168 for (j=0; j<num; j++) {
1169 for (i=0; i<argc-4; i++)
1170 append_file (i, name[i], size[i]);
1174 int main(int argc, char ** argv)
1178 char AGELOG_NAME[1024]= "/home/ningning/agefs.log";
1180 sprintf(cmd, "date >> %s", AGELOG_NAME);
1182 fplog = fopen(AGELOG_NAME, "a");
1184 for (i=0; i<argc; i++) {
1185 fprintf(fplog, "%s ", argv[i]);
1187 fprintf (fplog, "\n");
1189 if (argc>1 && (!strcmp(argv[1], "fmt2")))
1191 else if (argc>1 && (!strcmp(argv[1], "fmt3")))
1193 else if (argc>1 && (!strcmp(argv[1], "fmt4")))
1199 sprintf(cmd, "date >> %s", AGELOG_NAME);
1203 int main1(int argc, char ** argv)
1206 static int disk_index=0;
1207 int nfs3proc, size, off, count;
1212 int ftype_flag = 0, active_flag = 0;
1213 char name[MAX_PLAY_PATH_SIZE];
1217 profile_t create_profile, write_profile;
1224 EVEN_CHUNK_SIZE = atoi(argv[1]);
1225 RFS_ASSERT ((EVEN_CHUNK_SIZE==0) || (EVEN_CHUNK_SIZE==1));
1226 FILE_RATIO = atoi (argv[2]);
1227 ACTIVE_RATIO = atoi(argv[3]);
1228 WRITE_CHUNK_NUM = atoi(argv[4]);
1229 MAX_FS_SIZE_MB = atoi(argv[5]);
1231 if (WRITE_CHUNK_NUM==0)
1232 MIN_WRITE_SIZE = 2000000000;
1234 //MIN_WRITE_SIZE = DISK_FRAGMENT_SIZE;
1238 STAGE_NUM = atoi (argv[6]);
1239 strcpy (mapname, argv[7]);
1240 strcpy (testdir, argv[8]);
1241 ret = stat (testdir, &st);
1242 if ((ret == -1) && (errno==ENOENT)) {
1243 ret = mkdir (testdir, S_IRWXU);
1245 RFS_ASSERT (ret >= 0);
1248 /* add testdir to obj_fh */
1249 add_fh_t (&obj_fh, testdir, 1, -1, -1, IS_DIR, EXIST, ACTIVE);
1250 exist_active_obj_num ++;
1251 if (ACTIVE_RATIO >0)
1252 read_fh_map (mapname);
1254 print_fh_map(&obj_fh);
1255 init_profile_variables();
1257 fprintf(fplog, "main1: populate the file system with both trace files and randomly generated files\n");
1258 fprintf(fplog, "EVEN_CHUNK_SIZE %d FILE_RATIO %d ACTIVE_RATIO %d WRITE_CHUNK_NUM %d MAX_FS_SIZE_MB %d STAGE_NUM %d fh_map %s testdir %s\n", EVEN_CHUNK_SIZE, FILE_RATIO, ACTIVE_RATIO, WRITE_CHUNK_NUM, MAX_FS_SIZE_MB, STAGE_NUM, mapname, testdir);
1260 printf("EVEN_CHUNK_SIZE %d FILE_RATIO %d ACTIVE_RATIO %d WRITE_CHUNK_NUM %d MAX_FS_SIZE_MB %d STAGE_NUM %d fh_map %s testdir %s\n", EVEN_CHUNK_SIZE, FILE_RATIO, ACTIVE_RATIO, WRITE_CHUNK_NUM, MAX_FS_SIZE_MB, STAGE_NUM, mapname, testdir);
1264 init_profile ("create_profile", &create_profile);
1265 init_profile ("write_profile", &write_profile);
1267 start_profile (&create_profile);
1268 printf ("start creat/mkdir, active_obj_num %d\n", active_obj_num);
1269 for (i=0; (exist_active_obj_num <= active_obj_num) && (fs_size_MB < MAX_FS_SIZE_MB); i++) {
1271 if ((i!=0) && ((i%10000)==0)) {
1272 fprintf (stderr, "\n%d object created, exist_active_obj_num %d expected size %d MB\n", i, exist_active_obj_num, fs_size_MB);
1275 /* decide on the exact active obj or populated obj */
1276 if (decide(ACTIVE_RATIO)) {
1277 sfh = select_obj (&obj_fh, DONT_CARE, NON_EXIST, ACTIVE, 0, obj_fh.fh_max);
1281 obj_fh.fh[sfh].exist_flag = EXIST;
1282 exist_active_obj_num ++;
1283 ftype_flag = obj_fh.fh[sfh].ftype;
1284 size = obj_fh.fh[sfh].size;
1286 psfh = select_obj (&obj_fh, IS_DIR, EXIST, DONT_CARE, 0, obj_fh.fh_max);
1287 strcpy (name, obj_fh.fh[psfh].path);
1288 sfh = obj_fh.fh_max;
1289 sprintf(name+strlen(name), "/AGE%d", obj_fh.fh_max);
1291 /* decide next obj is file or directory */
1292 if (decide(FILE_RATIO)) {
1293 ftype_flag = IS_FILE;
1294 size = get_file_size();
1296 ftype_flag = IS_DIR;
1299 add_fh_t (&obj_fh, name, sfh, psfh, size, ftype_flag, EXIST, INACTIVE);
1303 /* make sure/create the obj pathname on disk */
1304 assure_exist (sfh, obj_fh.fh[sfh].path, ftype_flag);
1305 /* write file to sizes according certain distribution
1306 if (ftype_flag == IS_FILE)
1307 append_file (obj_fh.fh[sfh].path, obj_fh.fh[sfh].size);
1310 fprintf (stderr, "\n%d object created, exist_active_obj_num %d expected size %d MB\n", i, exist_active_obj_num, fs_size_MB);
1312 end_profile (&create_profile);
1313 start_profile (&write_profile);
1314 //write_file_range (0, obj_fh.fh_max);
1315 for (i=1; i<=STAGE_NUM; i++) {
1316 write_file_range (obj_fh.fh_max*(i-1)/STAGE_NUM, obj_fh.fh_max*i/STAGE_NUM);
1317 //fprintf(stderr, "getchar\n");
1320 end_profile (&write_profile);
1322 print_profile ("create_profile", &create_profile);
1323 print_profile ("write_profile", &write_profile);
1324 printf ("exist_active_obj_num %d active_obj_num %d fs_size_MB %d\n",
1325 exist_active_obj_num, active_obj_num, fs_size_MB);
1327 print_fh_map(&obj_fh);
1329 printf ("end of file system initialization\n");
1333 int write_file_range (int min, int max)
1336 int write_size, disk_write_size;
1339 printf ("start writing files min %d max %d\n", min, max);
1341 sfh = select_obj (&obj_fh, IS_FILE, EXIST, DONT_CARE, min, max);
1343 if (!decide(obj_fh.fh[sfh].size*100/(MIN_WRITE_SIZE*WRITE_CHUNK_NUM))) {
1344 printf("skip writing small file\n");
1350 write_size = get_write_size (obj_fh.fh[sfh].size, obj_fh.fh[sfh].cur_size);
1351 obj_fh.fh[sfh].cur_size += write_size;
1352 if (obj_fh.fh[sfh].cur_size == obj_fh.fh[sfh].size) {
1353 obj_fh.fh[sfh].exist_flag = COMPLETE;
1356 #define ACCUMULATE_SMALL_WRITE
1357 // This option improves speed by 12 times.
1358 #ifdef ACCUMULATE_SMALL_WRITE
1359 obj_fh.fh[sfh].accumulated_write_size += write_size;
1360 if (obj_fh.fh[sfh].exist_flag == COMPLETE) {
1361 disk_write_size = obj_fh.fh[sfh].accumulated_write_size;
1363 disk_write_size = (obj_fh.fh[sfh].accumulated_write_size -
1364 (obj_fh.fh[sfh].accumulated_write_size%DISK_FRAGMENT_SIZE));
1366 obj_fh.fh[sfh].accumulated_write_size -= disk_write_size;
1368 disk_write_size = write_size;
1371 if (disk_write_size >0) {
1372 append_file (sfh, obj_fh.fh[sfh].path, disk_write_size);
1374 printf ("%d C ", i);
1382 if ((j%100000)==0) {
1383 printf ("%d c ", j);