Logo Search packages:      
Sourcecode: lfc-postgres version File versions  Download package

delete_files.c

/*
 * Copyright (C) 1999-2004 by CERN/IT/PDP/DM
 * All rights reserved
 */
 
      /* Gets the times to delete a specified number of files from
       * the LFC, using a specified number of threads. The files are
       * initially created in separate directories for each thread,
       * then deleted. Individual delete times, time for each thread
       * to finish deleting and total time to finish deleting are
       * output. 
       */

      /* If dir is specified on the command line but does not start with
       * a slash, it is prefixed by $CASTOR_HOME..
       * Cnshost is set to the value of the environment variable CNS_HOST.
       * If not set, the value is taken from shift.conf.
       * If not set there either, use localhost.
       * If nb_threads is not specified, the program is single threaded.
       * Command syntax is:
       *    delete_files [-d dir] [-f number of files] [-t nb_threads] 
       *    [-r use relative pathnames] [-n nesting level of extra directories] 
       */
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <sys/time.h>
#include <sys/times.h>
#include <sys/types.h>
#include <uuid/uuid.h>
#if defined(_WIN32)
#include <winsock2.h>
#define F_OK 0
#else
#include <unistd.h>
#endif
#include "Cns_api.h"
#include "Cns.h"
#include "serrno.h"
#include "Cthread_api.h"
#define NFILES 10
#define MAX_FILES 999999
extern      char  *getenv();
extern      char  *optarg;

char Cnsdir[CA_MAXPATHLEN+1];
char toplevel[CA_MAXPATHLEN+1];
int nb_threads = 1;
int relative = 0;
int nb_files = NFILES;

main(argc, argv)
int argc;
char **argv;
{
      int c;
      char Cnshost[CA_MAXHOSTNAMELEN+1];
      char thread_dir[CA_MAXPATHLEN+1];
      char guid[CA_MAXGUIDLEN+1];
      char append[CA_MAXPATHLEN+1];
      uuid_t uuid;
      time_t current_time;
      char dir[CA_MAXPATHLEN+1];
      char subdir[CA_MAXPATHLEN+1];
      void *doit(void *);
      char *dp = NULL;
      char *endp;
      int errflg = 0;
      int dir_flag = 0;
      int depth = 0;
      int i,j,k;
      int current_numfiles = 0;
      char *p;
      char pid4print[11];
      struct Cns_filestat statbuf;
      int *tid;
      struct tm *tm = NULL;
      struct timeval utime;
      long start_time_us, end_time_us;
      char filename[CA_MAXPATHLEN+1];
      clock_t start_time_ms, end_time_ms;
#if defined(_WIN32)
      WSADATA wsadata;
#endif
      /* get command-line options */
      while ((c = getopt (argc, argv, "d:f:n:t:r")) != EOF) {
            switch (c) {
            case 'd':
                  strcpy(dir,optarg);
                  break;
            case 'f':
                  nb_files = strtol (optarg, &dp, 10);
                  if (*dp != '\0') {
                    fprintf (stderr, "invalid value for option -f\n");
                    errflg++;
                  }
                  break;
            case 'n':
                    depth = strtol (optarg, &dp, 10);
                  if (depth < 0) {
                          fprintf (stderr, "invalid value for option -n\n");
                        errflg++;
                  }
                  break;
            case 't':
                    nb_threads = strtol(optarg, &dp, 10);
                  if (nb_threads < 0) {
                    fprintf (stderr, "invalid value for option -n\n");
                    errflg++;
                  }
                  break;
            case 'r':
                    relative = 1;
                  break;
            }
      }

#if defined(_WIN32)
      if (WSAStartup (MAKEWORD (2, 0), &wsadata)) {
            fprintf (stderr, "WSAStartup unsuccessful\n");
            exit (SYERR);
      }
#endif

      /* set up directory name */
      sprintf (pid4print, "%d", getpid());
      if (dir) {
            if (*dir != '/') {
                  if ((p = getenv ("CASTOR_HOME")) == NULL ||
                      strlen (p) + strlen (dir) + strlen (pid4print) + 20 > CA_MAXPATHLEN) {
                        fprintf (stderr, "invalid value for option -d\n");
                        errflg++;
                  } else
                        sprintf (Cnsdir, "%s/%s", p, dir);
            } else {
                  if (strlen (dir) + strlen (pid4print) + 19 > CA_MAXPATHLEN) {
                        fprintf (stderr, "invalid value for option -d\n");
                        errflg++;
                  } else
                        strcpy (Cnsdir, dir);
            }
      } else {
            gethostname (Cnshost, sizeof(Cnshost));
            if ((p = getenv ("CASTOR_HOME")) == NULL ||
                strlen (p) + strlen (Cnshost) + strlen (pid4print) + 37 > CA_MAXPATHLEN) {
                  fprintf (stderr, "cannot set dir name\n");
                  errflg++;
            } else {
                  (void) time (&current_time);
                  tm = localtime (&current_time);
                  sprintf (Cnsdir, "%s/Cnstest/%s/%d%02d%02d", p, Cnshost,
                      tm->tm_year+1900, tm->tm_mon+1, tm->tm_mday);
            }
      }

      if (errflg) {
            fprintf (stderr, "usage: %s %s %s\n", argv[0],
                "[-d dir] [-f num_files] [-r relative pathname] [-t num_threads]");

#if defined(_WIN32)
            WSACleanup();
#endif
            exit (USERR);
      }
      
      sprintf(toplevel, "%s",Cnsdir);

      /* if extra directories were requested, add them to the
         pathname */
      if (depth) {
        for ( i = 1; i <= depth; i++) {
          sprintf(append,"/dir%d",i);
          strcat(Cnsdir,append);
        }
      }
      
      /* if the top level directory doesn't already exist, flag that
         you are going to create it */
      if (Cns_stat (toplevel, &statbuf) < 0) {
        dir_flag = 1;
      }

      /* create the directory if not there already */
      if (Cns_stat (Cnsdir, &statbuf) < 0) {
            if (serrno == ENOENT) {
                  endp = strrchr (Cnsdir, '/');
                  p = endp;
                  while (p > Cnsdir) {
                        *p = '\0';
                        c = Cns_access (Cnsdir, F_OK);
                        if (c == 0) break;
                        p = strrchr (Cnsdir, '/');
                  }
                  while (p <= endp) {
                        *p = '/';
                        c = Cns_mkdir (Cnsdir, 0777);
                        if (c < 0 && serrno != EEXIST) {
                              fprintf (stderr, "cannot create %s: %s\n",
                                  Cnsdir, sstrerror(serrno));
                              errflg++;
                              break;
                        }
                        p += strlen (p);
                  }
            } else {
                  fprintf (stderr, "%s: %s\n", Cnsdir, sstrerror(serrno));
                  errflg++;
            }
      }

      /* if multi-threaded, create an extra directory for each
           thread */
        if (nb_threads > 1) {
          for (i=0; i< nb_threads; i++) {
            sprintf(thread_dir, "%s", Cnsdir);
            sprintf(append, "/thread-%d", i);
            strcat(thread_dir, append);
            if (Cns_stat (thread_dir, &statbuf) <0) {               if (Cns_mkdir (thread_dir, 0777) < 0) {
            fprintf(stderr, "cannot create %s: %s\n",                                         thread_dir, sstrerror(serrno));
            errflg++;
            break;
              }
            }
          }
        }
      
      /* extra directories required if number of files is greater */
      /* than the maximum permitted per directory in the schema */
      if (nb_files > MAX_FILES) {
        for (j=0; j < nb_files; j+=MAX_FILES) {
          if (nb_threads > 1) {
            for (i=0; i< nb_threads; i++) {
            sprintf(subdir, "%s/thread-%d", Cnsdir, i);
            sprintf(append, "/%d", j);
            strcat(subdir, append);
            if (Cns_mkdir (subdir, 0777) < 0) {
              fprintf(stderr, "cannot create %s: %s\n",
                    subdir, sstrerror(serrno));
              errflg++;
              break;
            }
            }
          }
          else {
            sprintf(subdir, "%s", Cnsdir);
            sprintf(append, "/%d", j);
            strcat(subdir, append);
            if (Cns_mkdir (subdir, 0777) < 0) {
            fprintf(stderr, "cannot create %s: %s\n",
                  subdir, sstrerror(serrno));
            errflg++;
            break;
            }
          }
        }
      }

      /* create the files */
      for (j=0; j < nb_threads; j++) {
          strcpy(dir, Cnsdir);
          strcat(dir, "/");
          if (nb_threads > 1)
            sprintf(thread_dir, "thread-%d/",j);
          else sprintf(thread_dir, "");   
        if (nb_files > MAX_FILES) 
          strcat(thread_dir, "0/");
        strcat(dir,thread_dir);
          if (relative) {
            if ( Cns_chdir(dir) < 0) {
              fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
            }
          }
        for (i = 0; i < nb_files; i++) {
          if (k == MAX_FILES) {
            strcpy(dir, Cnsdir);
            strcat(dir, "/");
            if (nb_threads > 1 )
            strcat(dir,thread_dir);
            sprintf(subdir, "%d/", current_numfiles);
            strcat(dir, subdir);
            if (relative) {
            if ( Cns_chdir(dir) < 0) {
              fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
            }
            }
            k = 0;
          }
          sprintf (filename, "%s/file%d", dir, i);
          uuid_generate(uuid);
          uuid_unparse(uuid, guid);
          if (Cns_creatg (filename, guid, 0666) < 0) {
            fprintf (stderr, "%s: %s\n", filename, sstrerror(serrno));
            break;
          }
          current_numfiles++;
          k++;
        }
      }

      /* start all the threads going */
        gettimeofday( &utime, NULL);
        start_time_us = utime.tv_sec*1000000+utime.tv_usec;
        if (! errflg) {
        if ((tid = calloc (nb_threads, sizeof(int))) == NULL) {
          fprintf (stderr, "malloc error\n");
          errflg++;
        } else {
          for (i = 0; i < nb_threads; i++) {
            if ((tid[i] = Cthread_create (&doit, &i)) < 0) {
            fprintf (stderr, " error creating thread %d\n", i);
            errflg++;
            }
          }
          for (i = 0; i < nb_threads; i++) {
            (void)Cthread_join (tid[i], NULL);
          }
        }
        }
      gettimeofday( &utime, NULL);
        end_time_us = utime.tv_sec*1000000+utime.tv_usec;
      
      /* cd back to the base directory */
      sprintf(dir,"%s/../",toplevel);
      if (Cns_chdir(dir) < 0) {
        fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
      }

      /* remove all the directory structure that was created */

      strcpy(dir, Cnsdir);
      strcat(dir, "/");

      if (depth > 0) {
        /* recursively delete the extra directories */
        if (Cns_rmdir (Cnsdir) < 0) {
          fprintf (stderr, "%s: %s\n", Cnsdir, sstrerror(serrno));
        }
        for (i=depth-1; i>=1; i--) {
          sprintf(dir,"%s",toplevel);
          for (j=1; j<=i; j++) {
            sprintf(append, "/dir%d", j);
            strcat(dir, append);
          }
          if (Cns_rmdir (dir) < 0) {
            fprintf (stderr, "%s: %s\n", dir, sstrerror(serrno));
          }
        }
      }
      if (dir_flag != 0) {
        /* if you created the top level directory, remove it */
        if (Cns_rmdir (toplevel) < 0) {
          fprintf (stderr, "%s: %s\n", dir, sstrerror(serrno));
        }
      }
      if (errflg)
        exit (USERR);
        printf (" TOTAL %d\n", end_time_us - start_time_us);
      
      exit (0);
}

void *
doit(arg)
void *arg;
{
  char dir[CA_MAXPATHLEN+1];
  char thread_dir[CA_MAXPATHLEN+1];
  char subdir[CA_MAXPATHLEN+1];
  char append[CA_MAXPATHLEN+1];
  char filename[CA_MAXPATHLEN+1];
  struct timeval utime;
  long start_thread_time_us, end_thread_time_us;
  long start_run_time_us, end_run_time_us;
  int i, j, k = 0;
  int current_numfiles = 0;

  /* get correct directory for this thread */
  strcpy(dir, Cnsdir);
  strcat(dir, "/");
  
  if (nb_threads > 1 ){
    sprintf(thread_dir, "thread-%d/",*(int*)arg);
    strcat(dir,thread_dir);
  }
  if (nb_files > MAX_FILES) 
    strcat(dir, "0/");

  /* delete the files from this thread's directory */

  /* for chdir to the working directory if requested in command line */
  if (relative) {
    gettimeofday( &utime, NULL);
    start_thread_time_us = utime.tv_sec*1000000+utime.tv_usec;
    /* cd to the right directory */
    if (Cns_chdir(dir) < 0) {
      fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
    }
    for (i = 0; i < nb_files; i++) {
      if (k == MAX_FILES) {
      strcpy(dir, Cnsdir);
      strcat(dir, "/");
      if (nb_threads > 1 )
        strcat(dir,thread_dir);
      
      sprintf(subdir, "%d/", current_numfiles);
      strcat(dir, subdir);
      
      if ( Cns_chdir(dir) < 0) {
          fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
      }
      k = 0;
      }
      sprintf (filename, "file%d", i);
      /* start timer and do the delete */
      gettimeofday( &utime, NULL);
      start_run_time_us = utime.tv_sec*1000000+utime.tv_usec;
      if (Cns_unlink (filename) < 0) {
      fprintf (stderr, "%s: %s\n", filename, sstrerror(serrno));
      }
      /* end timer, print out result and update counters */
      gettimeofday( &utime, NULL);
      end_run_time_us = utime.tv_sec*1000000+utime.tv_usec;
      printf (" %d\n", end_run_time_us - start_run_time_us);
      current_numfiles++;
      k++;
    }
    
    gettimeofday( &utime, NULL);
    end_thread_time_us = utime.tv_sec*1000000+utime.tv_usec;
  }
  else {
    /* without chdir */
    gettimeofday( &utime, NULL);
    start_thread_time_us = utime.tv_sec*1000000+utime.tv_usec;
    for (i = 0; i < nb_files; i++) {
      if (k == MAX_FILES) {
      strcpy(dir, Cnsdir);
      strcat(dir, "/");
      if (nb_threads > 1 )
        strcat(dir,thread_dir);
      
      sprintf(subdir, "%d/", current_numfiles);
      strcat(dir, subdir);
        k = 0;
      }
      sprintf (filename, "%s/file%d", dir, i);
      /* start timer and do the delete*/
      gettimeofday( &utime, NULL);
      start_run_time_us = utime.tv_sec*1000000+utime.tv_usec;
      if (Cns_unlink (filename) < 0) {
      fprintf (stderr, "%s: %s\n", filename, sstrerror(serrno));
      }
      /* end timer, print result and update counters */
      gettimeofday( &utime, NULL);
      end_run_time_us = utime.tv_sec*1000000+utime.tv_usec;
      printf (" %d\n", end_run_time_us - start_run_time_us);
      current_numfiles++;
      k++;
    }
    gettimeofday( &utime, NULL);
    end_thread_time_us = utime.tv_sec*1000000+utime.tv_usec;
  }

  /* cd back to the base directory */
  sprintf(dir,"%s/../",toplevel);
  if (Cns_chdir(dir) < 0) {
    fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
  }

  /* remove the directory structure associated with this thread */
  if (nb_files > MAX_FILES) {
    for (j=0; j < nb_files; j+=MAX_FILES) {
      sprintf(subdir, "%s/%s", Cnsdir, thread_dir);
      sprintf(append, "/%d", j);
      strcat(subdir, append);
      if (Cns_rmdir (subdir) < 0) {
      fprintf(stderr, "cannot remove %s: %s\n",
            subdir, sstrerror(serrno));
      break;
      }
    }
  }
  if (nb_threads > 1) {
    sprintf(dir, "%s/%s", Cnsdir, thread_dir);
    if (Cns_rmdir (dir) < 0) {
      fprintf (stderr, "%s: %s\n", dir, sstrerror(serrno));
    }
  }
  printf (" THREAD %d\n", end_thread_time_us - start_thread_time_us);
}

Generated by  Doxygen 1.6.0   Back to index