version 1.2, 2004/07/08 17:27:59
|
version 1.4, 2005/03/23 12:12:08
|
Line 16 $|=1;
|
Line 16 $|=1;
|
# |
# |
|
|
# program version |
# program version |
my $version = "0.5.1 (8.7.2004)"; |
my $version = "0.6 (ROC 23.3.2005)"; |
|
|
|
# short help |
|
my $shorthelp = "MPIWG archiver $version\nuse:\n archiver [options] docpath\noptions:\n -premigrate don't delete archived files\n"; |
|
|
|
# read command line parameters |
|
my $args = MPIWGStor::parseargs; |
|
|
|
# debug level |
|
my $debug = (exists $$args{'debug'}) ? $$args{'debug'} : 0; |
|
|
# rewrite XML file (necessary for archive date!) |
# rewrite XML file (necessary for archive date!) |
my $fix_xml = 1; |
my $fix_xml = 1; |
Line 41 my $delete_data_files = 1;
|
Line 50 my $delete_data_files = 1;
|
# |
# |
my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc"; |
my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc"; |
if (! -x $archprog) { |
if (! -x $archprog) { |
print "ABORT: TSM client program '$archprog' missing!!\n"; |
logger('ABORT', "TSM client program '$archprog' missing!!\n"); |
exit 1; |
exit 1; |
} |
} |
my $checkprog = "/usr/local/mpiwg/archive/metacheck"; |
my $checkprog = "/usr/local/mpiwg/archive/metacheck"; |
if (! -x $checkprog) { |
if (! -x $checkprog) { |
print "ABORT: meta data checking program '$checkprog' missing!!\n"; |
logger('ABORT', "meta data checking program '$checkprog' missing!!\n"); |
exit 1; |
exit 1; |
} |
} |
# log file for archiver |
# log file for archiver |
my $log_file = "/var/log/mpiwg-archiver.log"; |
my $log_file = "/var/log/mpiwg-archiver.log"; |
if (! open LOG, ">>$log_file") { |
if (! open LOG, ">>$log_file") { |
print "ABORT: unable to write log file '$log_file'!!\n"; |
logger('ABORT', "unable to write log file '$log_file'!!\n"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 60 if (! open LOG, ">>$log_file") {
|
Line 69 if (! open LOG, ">>$log_file") {
|
# check parameters that were passed to the program |
# check parameters that were passed to the program |
# |
# |
if ($#ARGV < 0) { |
if ($#ARGV < 0) { |
print "ABORT: no document directory given!\n"; |
print $shorthelp; |
exit 1; |
exit 1; |
} |
} |
my $docdir = $ARGV[0]; |
my $docdir = $$args{'path'}; |
# strip double slashes |
# strip double slashes |
$docdir =~ s/\/\//\//; |
$docdir =~ s/\/\//\//; |
# strip trailing slashes |
# strip trailing slashes |
$docdir =~ s/\/+$//; |
$docdir =~ s/\/+$//; |
if (! -d $docdir) { |
if (! -d $docdir) { |
print "ABORT: document directory \'$docdir\' doesn't exist!\n"; |
logger('ABORT', "document directory \'$docdir\' doesn't exist!\n"); |
exit 1; |
exit 1; |
} |
} |
if (($#ARGV > 0)&&($ARGV[1] eq "-premigrate")) { |
# don't delete archived files with "-premigrate" |
$delete_data_files = 0; |
if (exists $$args{'premigrate'}) { |
|
$delete_data_files = not $$args{'premigrate'}; |
|
} |
|
if ($delete_data_files) { |
|
logger('INFO', "going to remove successfully archived files from disk"); |
} |
} |
|
|
my $metafile = "$docdir/index.meta"; |
my $metafile = "$docdir/index.meta"; |
if (! -f $metafile) { |
if (! -f $metafile) { |
print "ABORT: metadata index file \'$metafile\' doesn't exist!\n"; |
logger('ABORT', "metadata index file \'$metafile\' doesn't exist!\n"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 109 sub read_resource_meta {
|
Line 122 sub read_resource_meta {
|
# get archive-path |
# get archive-path |
$archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path')); |
$archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path')); |
if (! $archname) { |
if (! $archname) { |
print "ABORT: archive-name element missing!!\n"; |
logger('ABORT', "archive-name element missing!!\n"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 120 sub read_resource_meta {
|
Line 133 sub read_resource_meta {
|
foreach my $fn (@filenodes) { |
foreach my $fn (@filenodes) { |
my $name = MPIWGStor::sstrip($fn->findvalue('child::name')); |
my $name = MPIWGStor::sstrip($fn->findvalue('child::name')); |
my $path = MPIWGStor::sstrip($fn->findvalue('child::path')); |
my $path = MPIWGStor::sstrip($fn->findvalue('child::path')); |
print "FILE: ($path)$name\n"; |
logger('DEBUG', "FILE ($path)$name\n"); |
my $f = ($path) ? "$path/$name" : "$name"; |
my $f = ($path) ? "$path/$name" : "$name"; |
$files{$f} = $name; |
$files{$f} = $name; |
} |
} |
Line 130 sub read_resource_meta {
|
Line 143 sub read_resource_meta {
|
# |
# |
my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1); |
my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1); |
if ($stordatenode) { |
if ($stordatenode) { |
print "WARNING: archive storage date exists! Resource already archived?\n"; |
logger('WARNING', "archive storage date exists! Resource already archived?\n"); |
$warncnt++; |
$warncnt++; |
# delete old date |
# delete old date |
$stordatenode->removeChildNodes; |
$stordatenode->removeChildNodes; |
Line 173 sub run_archive {
|
Line 186 sub run_archive {
|
} |
} |
} |
} |
} else { |
} else { |
print "ABORT: unable to start archive command '$archcmd'!!\n"; |
logger('ABORT', "unable to start archive command '$archcmd'!!\n"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 196 sub check_files {
|
Line 209 sub check_files {
|
my $fp = "$docdir/$ft"; |
my $fp = "$docdir/$ft"; |
#print " fp: $fp\n"; |
#print " fp: $fp\n"; |
if ($$archived{$fp}) { |
if ($$archived{$fp}) { |
print "DEBUG: $ft archived OK\n"; |
logger('DEBUG', "$ft archived OK\n"); |
$$archived{$fp} = "OK"; |
$$archived{$fp} = "OK"; |
} else { |
} else { |
print "ERROR: file '$ft' missing from archive!\n"; |
logger('ERROR', "file '$ft' missing from archive!\n"); |
$errcnt++; |
$errcnt++; |
} |
} |
} |
} |
Line 208 sub check_files {
|
Line 221 sub check_files {
|
if ($$archived{$fa} ne "OK") { |
if ($$archived{$fa} ne "OK") { |
my ($fn, $fp) = MPIWGStor::split_file_path($fa); |
my ($fn, $fp) = MPIWGStor::split_file_path($fa); |
if ($MPIWGStor::index_files{$fn}) { |
if ($MPIWGStor::index_files{$fn}) { |
print "DEBUG: $fa ignored\n"; |
logger('DEBUG', "$fa ignored\n"); |
$na--; |
$na--; |
} else { |
} else { |
print "WARNING: $fa archived but not in list!\n"; |
logger('WARNING', "$fa archived but not in list!\n"); |
$warncnt++; |
$warncnt++; |
} |
} |
} |
} |
} |
} |
|
|
if ($nt > $na) { |
if ($nt > $na) { |
print "WARNING: less files were archived ($na vs. $nt)!\n"; |
logger('WARNING', "less files were archived ($na vs. $nt)!\n"); |
} elsif ($na > $nt) { |
} elsif ($na > $nt) { |
print "WARNING: more files were archived ($na vs. $nt)!\n"; |
logger('WARNING', "more files were archived ($na vs. $nt)!\n"); |
} |
} |
|
|
} |
} |
Line 248 sub delete_files {
|
Line 261 sub delete_files {
|
next unless (-f $f); |
next unless (-f $f); |
# delete files |
# delete files |
if (unlink $f) { |
if (unlink $f) { |
print "INFO: remove $f ($fn)\n"; |
logger('INFO', "remove $f ($fn)\n"); |
} else { |
} else { |
print "ERROR: unable to delete $f!\n"; |
logger('ERROR', "unable to delete $f!\n"); |
$errcnt++; |
$errcnt++; |
} |
} |
} |
} |
Line 262 sub delete_files {
|
Line 275 sub delete_files {
|
# dont't remove document dir (shouldn't be empty anyway) |
# dont't remove document dir (shouldn't be empty anyway) |
next if ($d eq $docdir); |
next if ($d eq $docdir); |
if (-d $d) { |
if (-d $d) { |
print "INFO: remove dir $d\n"; |
logger('INFO', "remove dir $d\n"); |
rmdir $d; |
rmdir $d; |
} |
} |
} |
} |
Line 279 sub delete_all_files {
|
Line 292 sub delete_all_files {
|
my ($files, $dir) = @_; |
my ($files, $dir) = @_; |
|
|
if (! opendir DIR, $dir) { |
if (! opendir DIR, $dir) { |
print "ERROR: unable to read directory $dir!\n"; |
logger('ERROR', "unable to read directory $dir!\n"); |
$errcnt++; |
$errcnt++; |
return; |
return; |
} |
} |
Line 293 sub delete_all_files {
|
Line 306 sub delete_all_files {
|
if (-f "$dir/$f") { |
if (-f "$dir/$f") { |
# $f is a file |
# $f is a file |
if (unlink "$dir/$f") { |
if (unlink "$dir/$f") { |
print "INFO: removed $f\n"; |
logger('INFO', "removed $f\n"); |
} else { |
} else { |
print "ERROR: unable to delete $f!\n"; |
logger('ERROR', "unable to delete $f!\n"); |
$errcnt++; |
$errcnt++; |
} |
} |
} elsif (-d _) { |
} elsif (-d _) { |
# $f is a directory (unlink won't work) |
# $f is a directory (unlink won't work) |
if ((system 'rm', '-r', "$dir/$f") == 0) { |
if ((system 'rm', '-r', "$dir/$f") == 0) { |
print "INFO: removed directory $f\n"; |
logger('INFO', "removed directory $f\n"); |
} else { |
} else { |
print "ERROR: unable to delete directory $f!\n"; |
logger('ERROR', "unable to delete directory $f!\n"); |
$errcnt++; |
$errcnt++; |
} |
} |
} else { |
} else { |
print "ERROR: funny object $dir/$f!\n"; |
logger('ERROR', "funny object $dir/$f!\n"); |
$errcnt++; |
$errcnt++; |
} |
} |
} else { |
} else { |
# $f is not in the list |
# $f is not in the list |
if (-d "$dir/$f") { |
if (-d "$dir/$f") { |
# recurse into directories |
# recurse into directories |
print "DEBUG: enter $dir/$f\n"; |
logger('DEBUG', "enter $dir/$f\n"); |
delete_all_files($files, "$dir/$f"); |
delete_all_files($files, "$dir/$f"); |
} |
} |
} |
} |
Line 326 sub delete_all_files {
|
Line 339 sub delete_all_files {
|
# main |
# main |
# |
# |
|
|
print "START: archiver $version at $archdate\n"; |
logger('START', "archiver $version at $archdate\n"); |
|
|
# make shure the right user is running this program |
# make shure the right user is running this program |
my $user = getlogin; |
my $user = getlogin; |
Line 337 my $user = getlogin;
|
Line 350 my $user = getlogin;
|
|
|
# use metacheck first |
# use metacheck first |
if (system("$checkprog $docdir >/dev/null") == 0) { |
if (system("$checkprog $docdir >/dev/null") == 0) { |
print "INFO: resource '$docdir' check OK\n"; |
logger('INFO', "resource '$docdir' check OK\n"); |
} else { |
} else { |
print "ABORT: resource '$docdir' check failed!!\n"; |
logger('ABORT', "resource '$docdir' check failed!!\n"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 354 print "INFO: ", scalar keys %$files_to_a
|
Line 367 print "INFO: ", scalar keys %$files_to_a
|
# check for .archived file |
# check for .archived file |
if (-f "$docdir/.archived") { |
if (-f "$docdir/.archived") { |
if (unlink "$docdir/.archived") { |
if (unlink "$docdir/.archived") { |
print "WARNING: existing .archived file has been removed! Resource already archived?\n"; |
logger('WARNING', "existing .archived file has been removed! Resource already archived?\n"); |
$warncnt++; |
$warncnt++; |
} else { |
} else { |
print "ERROR: unable to remove existing .archived file!\n"; |
logger('ERROR', "unable to remove existing .archived file!\n"); |
$errcnt++; |
$errcnt++; |
} |
} |
} |
} |
Line 369 if ($delete_junk_files) {
|
Line 382 if ($delete_junk_files) {
|
|
|
# write new index.meta |
# write new index.meta |
if ($errcnt > 0) { |
if ($errcnt > 0) { |
print "ABORT: there were errors!\n"; |
logger('ABORT', "there were errors!\n"); |
exit 1; |
exit 1; |
} else { |
} else { |
if ($fix_xml) { |
if ($fix_xml) { |
Line 378 if ($errcnt > 0) {
|
Line 391 if ($errcnt > 0) {
|
} |
} |
|
|
# start archiving |
# start archiving |
my $archived_files = run_archive; |
my $archived_files = run_archive(); |
|
my $num_archfiles = scalar keys %$archived_files; |
|
|
print "INFO: ", scalar keys %$archived_files, " files archived\n"; |
logger('INFO', "$num_archfiles files archived\n"); |
|
|
# check list of archived files |
# check list of archived files |
check_files($files_to_archive, $archived_files); |
check_files($files_to_archive, $archived_files); |
Line 398 if ($errcnt == 0) {
|
Line 412 if ($errcnt == 0) {
|
} |
} |
} |
} |
|
|
print "INFO: $warncnt warnings\n"; |
logger('INFO', "$warncnt warnings\n"); |
print "INFO: $errcnt errors\n"; |
logger('INFO', "$errcnt errors\n"); |
my $num_archfiles = %$archived_files + 1; |
|
if ($errcnt > 0) { |
if ($errcnt > 0) { |
print "ABORT: there were errors! ($num_archfiles files archived) at ", stime(time), "\n"; |
logger('ABORT', "there were errors! ($num_archfiles files archived) at " . stime(time)); |
exit 1; |
exit 1; |
} else { |
} else { |
print "DONE: $num_archfiles files archived at ", stime(time), "\n"; |
logger('DONE', "$num_archfiles files archived at " . stime(time)); |
} |
} |