Logo Search packages:      
Sourcecode: lfc-postgres version File versions  Download package

create_files.c

 /*
 * Copyright (C) 1999-2007 by CERN/IT/PDP/DM
 * All rights reserved
 */
 
#ifndef lint
static char sccsid[] = "@(#)$RCSfile: create_files.c,v $ $Revision: 1.6 $ $Date: 2007/05/24 09:46:44 $ CERN IT-PDP/DM Jean-Philippe Baud, Caitriana Nicholson";
#endif /* not lint */

      /* ask the name server running on Cnshost to create nb_threads * nb_files
       * files in a given directory dir using nb_threads threads
       */

      /* If dir is specified on the command line but does not start with
       * a slash, it is prefixed by $CASTOR_HOME.
       * If dir is not specified, the default is:
       *    $CASTOR_HOME/Cnstest/client_hostname/ccyymmdd
       * If nb_files is not given as parameter, the default value is NFILES.
       * Cnshost is set to the value of the environment variable CNS_HOST.
       * If not set, the value is taken from shift.conf.
       * If not set there either, use localhost.
       * If nb_threads is not specified, the program is single threaded.
       * Filenames are in the form:
       *    hhmmss_pid_lrand48()
       * Command syntax is:
       *    create_files [-d dir] [-f number_of_files] [-t number_of_threads]
       *                   [-n nesting_level] [-r use relative path]
       *                   [-x use transactions] [-c number of inserts per transaction]
       * If the -r option is specified, each thread will chdir to
       * the right directory before inserting the files.
       * If -n is used, there will be extra directories inserted
       * between the top level and the file. The number of extra
       * directories is set by nesting_level.
       * If -x is used, the creatg will be performed inside a transaction. The 
       * number of creates per transaction is controlled with the -c
       * option; otherwise, the default is 1.
       */
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <uuid/uuid.h>
#include <sys/times.h>
#include <sys/time.h>
#include <sys/types.h>
#if defined(_WIN32)
#include <winsock2.h>
#define F_OK 0
#else
#include <unistd.h>
#endif
#include "Cns_api.h"
#include "Cns.h"
#include "Cthread_api.h"
#include "serrno.h"
#define NFILES 10
#define MAX_FILES 999999
extern      char  *getenv();
extern      char  *optarg;
char filename[CA_MAXPATHLEN+1];
char Cnsdir[CA_MAXPATHLEN+1];
int nb_files = NFILES;
int nb_threads = 1;
int relative = 0;
int transactions = 0;
long commit_interval = 1;
main(argc, argv)
int argc;
char **argv;
{
      int c;
      char Cnshost[CA_MAXHOSTNAMELEN+1];
      char append[CA_MAXPATHLEN+1];
      char thread_dir[CA_MAXPATHLEN+1];
      char subdir[CA_MAXPATHLEN+1];
      char dir[CA_MAXPATHLEN+1];
      void *doit(void *);
      char *dp;
      char *endp;
      int errflg = 0;
      int i, j;
      char *p;
      char pid4print[11];
      struct Cns_filestat statbuf;
      int *tid;
      int depth = 0;
      int *arg;
      struct tm *tm = NULL;
      struct timeval utime;
      long start_time_us, end_time_us;
      time_t current_time;
#if defined(_WIN32)
      WSADATA wsadata;
#endif

      /* get command-line options */
      while ((c = getopt (argc, argv, "d:f:t:c:n:rx")) != EOF) {
            switch (c) {
            case 'd':
                  strcpy(dir, optarg);
                  break;
            case 'f':
                  nb_files = strtol (optarg, &dp, 10);
                  if (*dp != '\0') {
                        fprintf (stderr, "invalid value for option -f\n", nb_files);
                        errflg++;
                  }
                  break;
            case 't':
                  nb_threads = strtol (optarg, &dp, 10);
                  if (*dp != '\0' || nb_threads <= 0) {
                        fprintf (stderr, "invalid value for option -t\n");
                        errflg++;
                  }
                  break;
            case 'n':
                    depth = strtol (optarg, &dp, 0);
                  if (depth < 0) {
                          fprintf (stderr, "invalid value for option -n\n");
                        errflg++;
                  }
                  break;
            case 'r':
                    relative = 1;
                  break;
            case 'x':
                    transactions = 1;
                  break;
            case 'c':
                    commit_interval = strtol (optarg, &dp, 10);
                  if (*dp != '\0' || commit_interval <= 0) {
                        fprintf (stderr, "invalid value for option -c\n");
                        errflg++;
                  }
                  break;
            case '?':
                  errflg++;
                  break;
            default:
                  break;
            }
      }
#if defined(_WIN32)
      if (WSAStartup (MAKEWORD (2, 0), &wsadata)) {
            fprintf (stderr, "WSAStartup unsuccessful\n");
            exit (SYERR);
      }
#endif

      /* set up directory name according to chosen options */

      if (depth) {
        for ( i = 1; i <= depth; i++) {
          sprintf(append,"/%d",i);
          strcat(dir,append);
        }
      }

      sprintf (pid4print, "%d", getpid());
      if (dir) {
            if (*dir != '/') {
                  if ((p = getenv ("CASTOR_HOME")) == NULL ||
                      strlen (p) + strlen (dir) + strlen (pid4print) + 20 > CA_MAXPATHLEN) {
                        fprintf (stderr, "invalid value for option -d\n");
                        errflg++;
                  } else
                        sprintf (Cnsdir, "%s/%s", p, dir);
            } else {
                  if (strlen (dir) + strlen (pid4print) + 19 > CA_MAXPATHLEN) {
                        fprintf (stderr, "invalid value for option -d\n");
                        errflg++;
                  } else
                        strcpy (Cnsdir, dir);
            }
      } else {
            gethostname (Cnshost, sizeof(Cnshost));
            if ((p = getenv ("CASTOR_HOME")) == NULL ||
                strlen (p) + strlen (Cnshost) + strlen (pid4print) + 37 > CA_MAXPATHLEN) {
                  fprintf (stderr, "cannot set dir name\n");
                  errflg++;
            } else {
                  (void) time (&current_time);
                  tm = localtime (&current_time);
                  sprintf (Cnsdir, "%s/Cnstest/%s/%d%02d%02d/", p, Cnshost,
                      tm->tm_year+1900, tm->tm_mon+1, tm->tm_mday);
            }
      }
      if (errflg) {
            fprintf (stderr, "usage: %s %s\n", argv[0],
                "[-d dir] [-f number_of_files] [-t number_of_threads] [-n nesting_level] [-r use_relative pathnames] [-x use_transactions] [-c inserts_per_transaction]");
#if defined(_WIN32)
            WSACleanup();
#endif
            exit (USERR);
      }     

      /* create the directory if not there already */

      if (Cns_stat (Cnsdir, &statbuf) < 0) {
            if (serrno == ENOENT) {
                  endp = strrchr (Cnsdir, '/');
                  p = endp;
                  while (p > Cnsdir) {
                        *p = '\0';
                        c = Cns_access (Cnsdir, F_OK);
                        if (c == 0) break;
                        p = strrchr (Cnsdir, '/');
                  }
                  while (p <= endp) {
                        *p = '/';
                        c = Cns_mkdir (Cnsdir, 0777);
                        if (c < 0 && serrno != EEXIST) {
                              fprintf (stderr, "cannot create %s: %s\n",
                                  Cnsdir, sstrerror(serrno));
                              errflg++;
                              break;
                        }
                        p += strlen (p);
                  }
            } else {
                  fprintf (stderr, "%s: %s\n", Cnsdir, sstrerror(serrno));
                  errflg++;
            }
      }

      /* if multi-threaded, create a new directory for each thread */
      if (nb_threads > 1) {
        for (i=0; i< nb_threads; i++) {
          sprintf(thread_dir, "%s", Cnsdir);
          sprintf(append, "/thread-%d", i);
          strcat(thread_dir, append);
          if (Cns_stat (thread_dir, &statbuf) <0) {
            if (Cns_mkdir (thread_dir, 0777) < 0) {
            fprintf(stderr, "cannot create %s: %s\n",
                  thread_dir, sstrerror(serrno));
            errflg++;
            break;
            }
          }
        }
      }

      /* LFC schema has maximum of 999999 entries per directory */
      /* so create as many separate directories as are needed */
      if (nb_files > MAX_FILES) {
        for (j=0; j < nb_files; j+=MAX_FILES) {
          if (nb_threads > 1) {
            for (i=0; i< nb_threads; i++) {
            sprintf(subdir, "%s/thread-%d", Cnsdir, i);
            sprintf(append, "/%d", j);
            strcat(subdir, append);
            if (Cns_mkdir (subdir, 0777) < 0) {
              fprintf(stderr, "cannot create %s: %s\n",
                    subdir, sstrerror(serrno));
              errflg++;
              break;
            }
            }
          }
          else {
            sprintf(subdir, "%s", Cnsdir);
            sprintf(append, "/%d", j);
            strcat(subdir, append);
            if (Cns_mkdir (subdir, 0777) < 0) {
            fprintf(stderr, "cannot create %s: %s\n",
                  subdir, sstrerror(serrno));
            errflg++;
            break;
            }
          }
        }
      }

      /* allocate file name base */
      if (tm == NULL) {
            (void) time (&current_time);
            tm = localtime (&current_time);
      }

      sprintf (filename, "%02d%02d%02d_%d_",
               tm->tm_hour, tm->tm_min, tm->tm_sec, getpid());

      /* start the threads going */
      gettimeofday( &utime, NULL);
      start_time_us = utime.tv_sec*1000000+utime.tv_usec;
      
      if (nb_threads > 1) {
        if (! errflg) {
          if ((tid = calloc (nb_threads, sizeof(int))) == NULL) {
            fprintf (stderr, "malloc error\n");
            errflg++;
          } else {
            for (i = 0; i < nb_threads; i++) {
            arg = &i;
            if ((tid[i] = Cthread_create (&doit, arg)) < 0) {
              fprintf (stderr, " error creating thread %d\n", i);
              errflg++;
            }
            }
            for (i = 0; i < nb_threads; i++) {
            (void)Cthread_join (tid[i], NULL);
            }
          }
        }
      }
      else {
        i = 1;
        doit(&i);
      }
      
      gettimeofday( &utime, NULL);
      end_time_us = utime.tv_sec*1000000+utime.tv_usec;

      /* tidy up and finish */
#if defined(_WIN32)
      WSACleanup();
#endif
      if (errflg)
            exit (USERR);
      printf (" TOTAL %ld\n", end_time_us - start_time_us);
      
      exit (0);
}

void *
doit(arg)
void *arg;
{
      char fnbuf[CA_MAXPATHLEN+1];
      char thread_dir[CA_MAXPATHLEN+1];
      char dir[CA_MAXPATHLEN+1];
      char subdir[CA_MAXPATHLEN+1];
      int i,j,k;
      int current_numfiles = 0;
      char *p;
      char guid[CA_MAXGUIDLEN+1];
      int *Tidp = (int *)arg;
      struct timeval utime;
      uuid_t uuid;
      long start_thread_time_us, end_thread_time_us;
      long start_wait_time_us, end_wait_time_us;
      long start_run_time_us, end_run_time_us;
      long start_end_time_us, end_end_time_us;
      
      /* get the right directory for this thread */
      strcpy(dir, Cnsdir);
      strcat(dir, "/");
      
      if (nb_threads > 1 ){
        sprintf(thread_dir, "thread-%d/", *Tidp);
        strcat(dir,thread_dir);
      }
      if (nb_files > MAX_FILES) {
        sprintf(subdir, "%d/", 0);
        strcat(dir, subdir);
      }
      
      if (relative) {
        if ( Cns_chdir(dir) < 0) {
          fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
        }
        strcpy (fnbuf, filename);
      }
      else {
        strcpy(fnbuf, dir);
        strcat(fnbuf, filename);
      }
            
      /* pointer to the filename */
      p = fnbuf + strlen (fnbuf);
      
      gettimeofday( &utime, NULL);
      start_thread_time_us = utime.tv_sec*1000000+utime.tv_usec;
      
      /* loop through and create all the files, using transactions
         if option has been chosen */
      k=0;
      for (i=0; i < nb_files; i+=commit_interval) {
        if(transactions) {
          gettimeofday( &utime, NULL);
          start_wait_time_us = utime.tv_sec*1000000+utime.tv_usec;
          Cns_starttrans(NULL, "create_files");
          gettimeofday( &utime, NULL);
          end_wait_time_us = utime.tv_sec*1000000+utime.tv_usec;
        }
        gettimeofday( &utime, NULL);
        start_run_time_us = utime.tv_sec*1000000+utime.tv_usec;

        for (j = 0; j < commit_interval && j < nb_files; j++) {
          if (k == MAX_FILES) {
            strcpy(dir, Cnsdir);
            strcat(dir, "/");
            if (nb_threads > 1 )
            strcat(dir,thread_dir);

            sprintf(subdir, "%d/", current_numfiles);
            strcat(dir, subdir);

            if (relative) {
            if ( Cns_chdir(dir) < 0) {
              fprintf (stderr, "Cannot chdir %s: %s\n", Cnsdir, sstrerror(serrno));
            }
            strcpy (fnbuf, filename);
            }
            else {
            strcpy(fnbuf, dir);
            strcat(fnbuf, filename);
            }
            
            /* pointer to the filename */
            p = fnbuf + strlen (fnbuf);
            k = 0;
          }
#if defined(_WIN32)
          sprintf (p, "%d_%d", rand(), rand());
#else
          sprintf (p, "%d_%d", lrand48(), lrand48());
#endif
          uuid_generate(uuid);
          uuid_unparse(uuid, guid);
          
          if (Cns_creatg (fnbuf, guid, 0666) < 0) {
            fprintf (stderr, "%s: %s\n", fnbuf, sstrerror(serrno));
            if (transactions) Cns_aborttrans();
            break;
          }
          current_numfiles++;
          k++;
        }
        gettimeofday( &utime, NULL);
        end_run_time_us = utime.tv_sec*1000000+utime.tv_usec;
        if (transactions) {
          gettimeofday( &utime, NULL);
          start_end_time_us = utime.tv_sec*1000000+utime.tv_usec;
          Cns_endtrans();
          gettimeofday( &utime, NULL);
          end_end_time_us = utime.tv_sec*1000000+utime.tv_usec;
          printf (" %d %d %d\n", end_wait_time_us - start_wait_time_us, end_run_time_us - start_run_time_us, end_end_time_us - start_end_time_us);
        }
        else {
          printf (" %d\n", end_run_time_us - start_run_time_us);
        }
      }
      gettimeofday( &utime, NULL);
      end_thread_time_us = utime.tv_sec*1000000+utime.tv_usec;
      printf (" THREAD %ld\n", end_thread_time_us - start_thread_time_us);

      return (NULL);
}

Generated by  Doxygen 1.6.0   Back to index