Mercurial > hg > foxridge-archiver

comparison archiver.pl @ 11:b19df18aa19a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
updated command line option parsing and logging
author casties
date Wed, 23 Mar 2005 13:12:08 +0100
parents 1a377102b1ce
children fdf4ceb36db1
comparison
equal deleted inserted replaced
10:4417be0e2f07 11:b19df18aa19a
14 ####################################################### 14 #######################################################
15 # internal parameters 15 # internal parameters
16 # 16 #
17 17
18 # program version 18 # program version
19 my $version = "0.5.2 (12.7.2004)"; 19 my $version = "0.6 (ROC 23.3.2005)";
20
21 # short help
22 my $shorthelp = "MPIWG archiver $version\nuse:\n archiver [options] docpath\noptions:\n -premigrate don't delete archived files\n";
23
24 # read command line parameters
25 my $args = MPIWGStor::parseargs;
26
27 # debug level
28 my $debug = (exists $$args{'debug'}) ? $$args{'debug'} : 0;
20 29
21 # rewrite XML file (necessary for archive date!) 30 # rewrite XML file (necessary for archive date!)
22 my $fix_xml = 1; 31 my $fix_xml = 1;
23 my $xml_changed = 0; 32 my $xml_changed = 0;
24 # XML namespace (not really implemented!) 33 # XML namespace (not really implemented!)
39 ####################################################### 48 #######################################################
40 # external programs 49 # external programs
41 # 50 #
42 my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc"; 51 my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc";
43 if (! -x $archprog) { 52 if (! -x $archprog) {
44 print "ABORT: TSM client program '$archprog' missing!!\n"; 53 logger('ABORT', "TSM client program '$archprog' missing!!\n");
45 exit 1; 54 exit 1;
46 } 55 }
47 my $checkprog = "/usr/local/mpiwg/archive/metacheck"; 56 my $checkprog = "/usr/local/mpiwg/archive/metacheck";
48 if (! -x $checkprog) { 57 if (! -x $checkprog) {
49 print "ABORT: meta data checking program '$checkprog' missing!!\n"; 58 logger('ABORT', "meta data checking program '$checkprog' missing!!\n");
50 exit 1; 59 exit 1;
51 } 60 }
52 # log file for archiver 61 # log file for archiver
53 my $log_file = "/var/log/mpiwg-archiver.log"; 62 my $log_file = "/var/log/mpiwg-archiver.log";
54 if (! open LOG, ">>$log_file") { 63 if (! open LOG, ">>$log_file") {
55 print "ABORT: unable to write log file '$log_file'!!\n"; 64 logger('ABORT', "unable to write log file '$log_file'!!\n");
56 exit 1; 65 exit 1;
57 } 66 }
58 67
59 ####################################################### 68 #######################################################
60 # check parameters that were passed to the program 69 # check parameters that were passed to the program
61 # 70 #
62 if ($#ARGV < 0) { 71 if ($#ARGV < 0) {
63 print "ABORT: no document directory given!\n"; 72 print $shorthelp;
64 exit 1; 73 exit 1;
65 } 74 }
66 my $docdir = $ARGV[0]; 75 my $docdir = $$args{'path'};
67 # strip double slashes 76 # strip double slashes
68 $docdir =~ s/\/\//\//; 77 $docdir =~ s/\/\//\//;
69 # strip trailing slashes 78 # strip trailing slashes
70 $docdir =~ s/\/+$//; 79 $docdir =~ s/\/+$//;
71 if (! -d $docdir) { 80 if (! -d $docdir) {
72 print "ABORT: document directory \'$docdir\' doesn't exist!\n"; 81 logger('ABORT', "document directory \'$docdir\' doesn't exist!\n");
73 exit 1; 82 exit 1;
74 } 83 }
75 if (($#ARGV > 0)&&($ARGV[1] eq "-premigrate")) { 84 # don't delete archived files with "-premigrate"
76 $delete_data_files = 0; 85 if (exists $$args{'premigrate'}) {
86 $delete_data_files = not $$args{'premigrate'};
87 }
88 if ($delete_data_files) {
89 logger('INFO', "going to remove successfully archived files from disk");
77 } 90 }
78 91
79 my $metafile = "$docdir/index.meta"; 92 my $metafile = "$docdir/index.meta";
80 if (! -f $metafile) { 93 if (! -f $metafile) {
81 print "ABORT: metadata index file \'$metafile\' doesn't exist!\n"; 94 logger('ABORT', "metadata index file \'$metafile\' doesn't exist!\n");
82 exit 1; 95 exit 1;
83 } 96 }
84 97
85 ####################################################### 98 #######################################################
86 # internal variables 99 # internal variables
107 # archive path 120 # archive path
108 # 121 #
109 # get archive-path 122 # get archive-path
110 $archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path')); 123 $archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path'));
111 if (! $archname) { 124 if (! $archname) {
112 print "ABORT: archive-name element missing!!\n"; 125 logger('ABORT', "archive-name element missing!!\n");
113 exit 1; 126 exit 1;
114 } 127 }
115 128
116 # 129 #
117 # files 130 # files
118 # 131 #
119 my @filenodes = $rootnode->findnodes('child::file'); 132 my @filenodes = $rootnode->findnodes('child::file');
120 foreach my $fn (@filenodes) { 133 foreach my $fn (@filenodes) {
121 my $name = MPIWGStor::sstrip($fn->findvalue('child::name')); 134 my $name = MPIWGStor::sstrip($fn->findvalue('child::name'));
122 my $path = MPIWGStor::sstrip($fn->findvalue('child::path')); 135 my $path = MPIWGStor::sstrip($fn->findvalue('child::path'));
123 print "FILE: ($path)$name\n"; 136 logger('DEBUG', "FILE ($path)$name\n");
124 my $f = ($path) ? "$path/$name" : "$name"; 137 my $f = ($path) ? "$path/$name" : "$name";
125 $files{$f} = $name; 138 $files{$f} = $name;
126 } 139 }
127 140
128 # 141 #
129 # archive-storage-date 142 # archive-storage-date
130 # 143 #
131 my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1); 144 my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1);
132 if ($stordatenode) { 145 if ($stordatenode) {
133 print "WARNING: archive storage date exists! Resource already archived?\n"; 146 logger('WARNING', "archive storage date exists! Resource already archived?\n");
134 $warncnt++; 147 $warncnt++;
135 # delete old date 148 # delete old date
136 $stordatenode->removeChildNodes; 149 $stordatenode->removeChildNodes;
137 } else { 150 } else {
138 # create new storage date node 151 # create new storage date node
171 if (/^Archive processing of .* finished without failure./) { 184 if (/^Archive processing of .* finished without failure./) {
172 print " ARCH: OK\n"; 185 print " ARCH: OK\n";
173 } 186 }
174 } 187 }
175 } else { 188 } else {
176 print "ABORT: unable to start archive command '$archcmd'!!\n"; 189 logger('ABORT', "unable to start archive command '$archcmd'!!\n");
177 exit 1; 190 exit 1;
178 } 191 }
179 192
180 return \%files; 193 return \%files;
181 } 194 }
194 207
195 foreach my $ft (sort keys %$to_archive) { 208 foreach my $ft (sort keys %$to_archive) {
196 my $fp = "$docdir/$ft"; 209 my $fp = "$docdir/$ft";
197 #print " fp: $fp\n"; 210 #print " fp: $fp\n";
198 if ($$archived{$fp}) { 211 if ($$archived{$fp}) {
199 print "DEBUG: $ft archived OK\n"; 212 logger('DEBUG', "$ft archived OK\n");
200 $$archived{$fp} = "OK"; 213 $$archived{$fp} = "OK";
201 } else { 214 } else {
202 print "ERROR: file '$ft' missing from archive!\n"; 215 logger('ERROR', "file '$ft' missing from archive!\n");
203 $errcnt++; 216 $errcnt++;
204 } 217 }
205 } 218 }
206 219
207 foreach my $fa (sort keys %$archived) { 220 foreach my $fa (sort keys %$archived) {
208 if ($$archived{$fa} ne "OK") { 221 if ($$archived{$fa} ne "OK") {
209 my ($fn, $fp) = MPIWGStor::split_file_path($fa); 222 my ($fn, $fp) = MPIWGStor::split_file_path($fa);
210 if ($MPIWGStor::index_files{$fn}) { 223 if ($MPIWGStor::index_files{$fn}) {
211 print "DEBUG: $fa ignored\n"; 224 logger('DEBUG', "$fa ignored\n");
212 $na--; 225 $na--;
213 } else { 226 } else {
214 print "WARNING: $fa archived but not in list!\n"; 227 logger('WARNING', "$fa archived but not in list!\n");
215 $warncnt++; 228 $warncnt++;
216 } 229 }
217 } 230 }
218 } 231 }
219 232
220 if ($nt > $na) { 233 if ($nt > $na) {
221 print "WARNING: less files were archived ($na vs. $nt)!\n"; 234 logger('WARNING', "less files were archived ($na vs. $nt)!\n");
222 } elsif ($na > $nt) { 235 } elsif ($na > $nt) {
223 print "WARNING: more files were archived ($na vs. $nt)!\n"; 236 logger('WARNING', "more files were archived ($na vs. $nt)!\n");
224 } 237 }
225 238
226 } 239 }
227 240
228 241
246 next if ($MPIWGStor::index_files{$fn}); 259 next if ($MPIWGStor::index_files{$fn});
247 # no file no delete 260 # no file no delete
248 next unless (-f $f); 261 next unless (-f $f);
249 # delete files 262 # delete files
250 if (unlink $f) { 263 if (unlink $f) {
251 print "INFO: remove $f ($fn)\n"; 264 logger('INFO', "remove $f ($fn)\n");
252 } else { 265 } else {
253 print "ERROR: unable to delete $f!\n"; 266 logger('ERROR', "unable to delete $f!\n");
254 $errcnt++; 267 $errcnt++;
255 } 268 }
256 } 269 }
257 # try to delete all empty directories 270 # try to delete all empty directories
258 my @dirkeys = sort keys %dirs; 271 my @dirkeys = sort keys %dirs;
260 for (my $i = $#dirkeys; $i >= 0; $i--) { 273 for (my $i = $#dirkeys; $i >= 0; $i--) {
261 my $d = $dirkeys[$i]; 274 my $d = $dirkeys[$i];
262 # dont't remove document dir (shouldn't be empty anyway) 275 # dont't remove document dir (shouldn't be empty anyway)
263 next if ($d eq $docdir); 276 next if ($d eq $docdir);
264 if (-d $d) { 277 if (-d $d) {
265 print "INFO: remove dir $d\n"; 278 logger('INFO', "remove dir $d\n");
266 rmdir $d; 279 rmdir $d;
267 } 280 }
268 } 281 }
269 } 282 }
270 283
277 # 290 #
278 sub delete_all_files { 291 sub delete_all_files {
279 my ($files, $dir) = @_; 292 my ($files, $dir) = @_;
280 293
281 if (! opendir DIR, $dir) { 294 if (! opendir DIR, $dir) {
282 print "ERROR: unable to read directory $dir!\n"; 295 logger('ERROR', "unable to read directory $dir!\n");
283 $errcnt++; 296 $errcnt++;
284 return; 297 return;
285 } 298 }
286 my @fl = readdir DIR; 299 my @fl = readdir DIR;
287 closedir DIR; 300 closedir DIR;
291 if ($$files{$f}) { 304 if ($$files{$f}) {
292 # $f is in the file list 305 # $f is in the file list
293 if (-f "$dir/$f") { 306 if (-f "$dir/$f") {
294 # $f is a file 307 # $f is a file
295 if (unlink "$dir/$f") { 308 if (unlink "$dir/$f") {
296 print "INFO: removed $f\n"; 309 logger('INFO', "removed $f\n");
297 } else { 310 } else {
298 print "ERROR: unable to delete $f!\n"; 311 logger('ERROR', "unable to delete $f!\n");
299 $errcnt++; 312 $errcnt++;
300 } 313 }
301 } elsif (-d _) { 314 } elsif (-d _) {
302 # $f is a directory (unlink won't work) 315 # $f is a directory (unlink won't work)
303 if ((system 'rm', '-r', "$dir/$f") == 0) { 316 if ((system 'rm', '-r', "$dir/$f") == 0) {
304 print "INFO: removed directory $f\n"; 317 logger('INFO', "removed directory $f\n");
305 } else { 318 } else {
306 print "ERROR: unable to delete directory $f!\n"; 319 logger('ERROR', "unable to delete directory $f!\n");
307 $errcnt++; 320 $errcnt++;
308 } 321 }
309 } else { 322 } else {
310 print "ERROR: funny object $dir/$f!\n"; 323 logger('ERROR', "funny object $dir/$f!\n");
311 $errcnt++; 324 $errcnt++;
312 } 325 }
313 } else { 326 } else {
314 # $f is not in the list 327 # $f is not in the list
315 if (-d "$dir/$f") { 328 if (-d "$dir/$f") {
316 # recurse into directories 329 # recurse into directories
317 print "DEBUG: enter $dir/$f\n"; 330 logger('DEBUG', "enter $dir/$f\n");
318 delete_all_files($files, "$dir/$f"); 331 delete_all_files($files, "$dir/$f");
319 } 332 }
320 } 333 }
321 } 334 }
322 } 335 }
324 337
325 ####################################################### 338 #######################################################
326 # main 339 # main
327 # 340 #
328 341
329 print "START: archiver $version at $archdate\n"; 342 logger('START', "archiver $version at $archdate\n");
330 343
331 # make shure the right user is running this program 344 # make shure the right user is running this program
332 my $user = getlogin; 345 my $user = getlogin;
333 #if (($user ne "archive")||($user ne "root")) { 346 #if (($user ne "archive")||($user ne "root")) {
334 # logger("ABORT", "you ($user) must be archive or root user to run this program!"); 347 # logger("ABORT", "you ($user) must be archive or root user to run this program!");
335 # exit 1; 348 # exit 1;
336 #} 349 #}
337 350
338 # use metacheck first 351 # use metacheck first
339 if (system("$checkprog $docdir >/dev/null") == 0) { 352 if (system("$checkprog $docdir >/dev/null") == 0) {
340 print "INFO: resource '$docdir' check OK\n"; 353 logger('INFO', "resource '$docdir' check OK\n");
341 } else { 354 } else {
342 print "ABORT: resource '$docdir' check failed!!\n"; 355 logger('ABORT', "resource '$docdir' check failed!!\n");
343 exit 1; 356 exit 1;
344 } 357 }
345 358
346 # read index.meta file 359 # read index.meta file
347 my ($document, $rootnode) = MPIWGStor::read_xml($metafile); 360 my ($document, $rootnode) = MPIWGStor::read_xml($metafile);
352 print "INFO: ", scalar keys %$files_to_archive, " files to archive\n"; 365 print "INFO: ", scalar keys %$files_to_archive, " files to archive\n";
353 366
354 # check for .archived file 367 # check for .archived file
355 if (-f "$docdir/.archived") { 368 if (-f "$docdir/.archived") {
356 if (unlink "$docdir/.archived") { 369 if (unlink "$docdir/.archived") {
357 print "WARNING: existing .archived file has been removed! Resource already archived?\n"; 370 logger('WARNING', "existing .archived file has been removed! Resource already archived?\n");
358 $warncnt++; 371 $warncnt++;
359 } else { 372 } else {
360 print "ERROR: unable to remove existing .archived file!\n"; 373 logger('ERROR', "unable to remove existing .archived file!\n");
361 $errcnt++; 374 $errcnt++;
362 } 375 }
363 } 376 }
364 377
365 # remove junk files 378 # remove junk files
367 delete_all_files(\%MPIWGStor::junk_files, $docdir); 380 delete_all_files(\%MPIWGStor::junk_files, $docdir);
368 } 381 }
369 382
370 # write new index.meta 383 # write new index.meta
371 if ($errcnt > 0) { 384 if ($errcnt > 0) {
372 print "ABORT: there were errors!\n"; 385 logger('ABORT', "there were errors!\n");
373 exit 1; 386 exit 1;
374 } else { 387 } else {
375 if ($fix_xml) { 388 if ($fix_xml) {
376 MPIWGStor::write_xml($document, $metafile); 389 MPIWGStor::write_xml($document, $metafile);
377 } 390 }
379 392
380 # start archiving 393 # start archiving
381 my $archived_files = run_archive(); 394 my $archived_files = run_archive();
382 my $num_archfiles = scalar keys %$archived_files; 395 my $num_archfiles = scalar keys %$archived_files;
383 396
384 print "INFO: $num_archfiles files archived\n"; 397 logger('INFO', "$num_archfiles files archived\n");
385 398
386 # check list of archived files 399 # check list of archived files
387 check_files($files_to_archive, $archived_files); 400 check_files($files_to_archive, $archived_files);
388 401
389 # delete files if all went OK 402 # delete files if all went OK
397 if ($delete_data_files) { 410 if ($delete_data_files) {
398 delete_files($archived_files); 411 delete_files($archived_files);
399 } 412 }
400 } 413 }
401 414
402 print "INFO: $warncnt warnings\n"; 415 logger('INFO', "$warncnt warnings\n");
403 print "INFO: $errcnt errors\n"; 416 logger('INFO', "$errcnt errors\n");
404 if ($errcnt > 0) { 417 if ($errcnt > 0) {
405 print "ABORT: there were errors! ($num_archfiles files archived) at ", stime(time), "\n"; 418 logger('ABORT', "there were errors! ($num_archfiles files archived) at " . stime(time));
406 exit 1; 419 exit 1;
407 } else { 420 } else {
408 print "DONE: $num_archfiles files archived at ", stime(time), "\n"; 421 logger('DONE', "$num_archfiles files archived at " . stime(time));
409 } 422 }