version 1.4, 2005/03/23 12:12:08
|
version 1.7, 2005/10/05 13:35:12
|
Line 16 $|=1;
|
Line 16 $|=1;
|
# |
# |
|
|
# program version |
# program version |
my $version = "0.6 (ROC 23.3.2005)"; |
my $version = "0.7.2 (ROC 5.10.2005)"; |
|
|
# short help |
# short help |
my $shorthelp = "MPIWG archiver $version\nuse:\n archiver [options] docpath\noptions:\n -premigrate don't delete archived files\n"; |
my $help = "MPIWG archiver $version |
|
use: archiver [options] docpath |
|
options: |
|
-debug show debugging info |
|
-premigrate don't delete archived files |
|
-force archive even if already archived |
|
-replace rewrite index file |
|
"; |
|
|
# read command line parameters |
# read command line parameters |
my $args = MPIWGStor::parseargs; |
my $args = MPIWGStor::parseargs; |
|
if (! scalar(%$args)) { |
|
print $help, "\n"; |
|
exit 1; |
|
} |
|
|
# debug level |
# debug level |
my $debug = (exists $$args{'debug'}) ? $$args{'debug'} : 0; |
$debug = (exists $$args{'debug'}) ? $$args{'debug'} : 0; |
|
|
|
# force archiving |
|
my $force_archive = (exists $$args{'force'}) ? $$args{'force'} : 0; |
|
|
|
# rewrite index file |
|
my $rewrite_index = (exists $$args{'replace'}) ? $$args{'replace'} : 0; |
|
|
# rewrite XML file (necessary for archive date!) |
# rewrite XML file (necessary for archive date!) |
my $fix_xml = 1; |
my $fix_xml = 1; |
my $xml_changed = 0; |
my $xml_changed = 0; |
|
|
# XML namespace (not really implemented!) |
# XML namespace (not really implemented!) |
my $namespace = ""; |
my $namespace = ""; |
|
|
# archive name (archive-path element, usually == $docdir) |
# archive name (archive-path element, usually == $docdir) |
my $archname; |
my $archname; |
|
|
# archive storage date (now) |
# archive storage date (now) |
my $archdate = stime(time); |
my $archdate = stime(time); |
|
|
Line 44 my $delete_junk_files = 1;
|
Line 63 my $delete_junk_files = 1;
|
# delete data files after archiving |
# delete data files after archiving |
my $delete_data_files = 1; |
my $delete_data_files = 1; |
|
|
|
# don't delete archived files with "-premigrate" |
|
if (exists $$args{'premigrate'}) { |
|
$delete_data_files = not $$args{'premigrate'}; |
|
} |
|
if ($delete_data_files) { |
|
logger('INFO', "going to remove successfully archived files from disk"); |
|
} |
|
|
|
|
####################################################### |
####################################################### |
# external programs |
# external programs |
# |
# |
my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc"; |
my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc"; |
if (! -x $archprog) { |
if (! -x $archprog) { |
logger('ABORT', "TSM client program '$archprog' missing!!\n"); |
logger('ABORT', "TSM client program '$archprog' missing!"); |
exit 1; |
exit 1; |
} |
} |
my $checkprog = "/usr/local/mpiwg/archive/metacheck"; |
my $checkprog = "/usr/local/mpiwg/archive/metacheck"; |
if (! -x $checkprog) { |
if (! -x $checkprog) { |
logger('ABORT', "meta data checking program '$checkprog' missing!!\n"); |
logger('ABORT', "meta data checking program '$checkprog' missing!"); |
exit 1; |
exit 1; |
} |
} |
# log file for archiver |
# log file for archiver |
my $log_file = "/var/log/mpiwg-archiver.log"; |
my $log_file = "/var/log/mpiwg-archiver.log"; |
if (! open LOG, ">>$log_file") { |
if (! open LOG, ">>$log_file") { |
logger('ABORT', "unable to write log file '$log_file'!!\n"); |
logger('ABORT', "unable to write log file '$log_file'!"); |
exit 1; |
exit 1; |
} |
} |
|
|
####################################################### |
####################################################### |
# check parameters that were passed to the program |
# check parameters that were passed to the program |
# |
# |
if ($#ARGV < 0) { |
|
print $shorthelp; |
|
exit 1; |
|
} |
|
my $docdir = $$args{'path'}; |
my $docdir = $$args{'path'}; |
# strip double slashes |
# strip double slashes |
$docdir =~ s/\/\//\//; |
$docdir =~ s/\/\//\//; |
# strip trailing slashes |
# strip trailing slashes |
$docdir =~ s/\/+$//; |
$docdir =~ s/\/+$//; |
if (! -d $docdir) { |
if (! -d $docdir) { |
logger('ABORT', "document directory \'$docdir\' doesn't exist!\n"); |
logger('ABORT', "document directory \'$docdir\' doesn't exist!"); |
exit 1; |
exit 1; |
} |
} |
# don't delete archived files with "-premigrate" |
|
if (exists $$args{'premigrate'}) { |
|
$delete_data_files = not $$args{'premigrate'}; |
|
} |
|
if ($delete_data_files) { |
|
logger('INFO', "going to remove successfully archived files from disk"); |
|
} |
|
|
|
my $metafile = "$docdir/index.meta"; |
my $metafile = "$docdir/index.meta"; |
if (! -f $metafile) { |
if (! -f $metafile) { |
logger('ABORT', "metadata index file \'$metafile\' doesn't exist!\n"); |
logger('ABORT', "metadata index file \'$metafile\' doesn't exist!"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 122 sub read_resource_meta {
|
Line 139 sub read_resource_meta {
|
# get archive-path |
# get archive-path |
$archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path')); |
$archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path')); |
if (! $archname) { |
if (! $archname) { |
logger('ABORT', "archive-name element missing!!\n"); |
logger('ABORT', "archive-name element missing!"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 133 sub read_resource_meta {
|
Line 150 sub read_resource_meta {
|
foreach my $fn (@filenodes) { |
foreach my $fn (@filenodes) { |
my $name = MPIWGStor::sstrip($fn->findvalue('child::name')); |
my $name = MPIWGStor::sstrip($fn->findvalue('child::name')); |
my $path = MPIWGStor::sstrip($fn->findvalue('child::path')); |
my $path = MPIWGStor::sstrip($fn->findvalue('child::path')); |
logger('DEBUG', "FILE ($path)$name\n"); |
logger('DEBUG', "FILE ($path)$name"); |
my $f = ($path) ? "$path/$name" : "$name"; |
my $f = ($path) ? "$path/$name" : "$name"; |
$files{$f} = $name; |
$files{$f} = $name; |
} |
} |
Line 143 sub read_resource_meta {
|
Line 160 sub read_resource_meta {
|
# |
# |
my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1); |
my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1); |
if ($stordatenode) { |
if ($stordatenode) { |
logger('WARNING', "archive storage date exists! Resource already archived?\n"); |
logger('WARNING', "archive storage date exists! Resource already archived?"); |
$warncnt++; |
$warncnt++; |
# delete old date |
# delete old date |
$stordatenode->removeChildNodes; |
$stordatenode->removeChildNodes; |
Line 186 sub run_archive {
|
Line 203 sub run_archive {
|
} |
} |
} |
} |
} else { |
} else { |
logger('ABORT', "unable to start archive command '$archcmd'!!\n"); |
logger('ABORT', "unable to start archive command '$archcmd'!!"); |
exit 1; |
exit 1; |
} |
} |
|
|
Line 209 sub check_files {
|
Line 226 sub check_files {
|
my $fp = "$docdir/$ft"; |
my $fp = "$docdir/$ft"; |
#print " fp: $fp\n"; |
#print " fp: $fp\n"; |
if ($$archived{$fp}) { |
if ($$archived{$fp}) { |
logger('DEBUG', "$ft archived OK\n"); |
logger('DEBUG', "$ft archived OK"); |
$$archived{$fp} = "OK"; |
$$archived{$fp} = "OK"; |
} else { |
} else { |
logger('ERROR', "file '$ft' missing from archive!\n"); |
logger('ERROR', "file '$ft' missing from archive!"); |
$errcnt++; |
$errcnt++; |
} |
} |
} |
} |
Line 221 sub check_files {
|
Line 238 sub check_files {
|
if ($$archived{$fa} ne "OK") { |
if ($$archived{$fa} ne "OK") { |
my ($fn, $fp) = MPIWGStor::split_file_path($fa); |
my ($fn, $fp) = MPIWGStor::split_file_path($fa); |
if ($MPIWGStor::index_files{$fn}) { |
if ($MPIWGStor::index_files{$fn}) { |
logger('DEBUG', "$fa ignored\n"); |
logger('DEBUG', "$fa ignored"); |
$na--; |
$na--; |
} else { |
} else { |
logger('WARNING', "$fa archived but not in list!\n"); |
logger('WARNING', "$fa archived but not in list!"); |
$warncnt++; |
$warncnt++; |
} |
} |
} |
} |
} |
} |
|
|
if ($nt > $na) { |
if ($nt > $na) { |
logger('WARNING', "less files were archived ($na vs. $nt)!\n"); |
logger('WARNING', "less files were archived ($na vs. $nt)!"); |
} elsif ($na > $nt) { |
} elsif ($na > $nt) { |
logger('WARNING', "more files were archived ($na vs. $nt)!\n"); |
logger('WARNING', "more files were archived ($na vs. $nt)!"); |
} |
} |
|
|
} |
} |
Line 261 sub delete_files {
|
Line 278 sub delete_files {
|
next unless (-f $f); |
next unless (-f $f); |
# delete files |
# delete files |
if (unlink $f) { |
if (unlink $f) { |
logger('INFO', "remove $f ($fn)\n"); |
logger('DEBUG', "remove $f ($fn)"); |
} else { |
} else { |
logger('ERROR', "unable to delete $f!\n"); |
logger('ERROR', "unable to delete $f!"); |
$errcnt++; |
$errcnt++; |
} |
} |
} |
} |
Line 275 sub delete_files {
|
Line 292 sub delete_files {
|
# dont't remove document dir (shouldn't be empty anyway) |
# dont't remove document dir (shouldn't be empty anyway) |
next if ($d eq $docdir); |
next if ($d eq $docdir); |
if (-d $d) { |
if (-d $d) { |
logger('INFO', "remove dir $d\n"); |
logger('DEBUG', "remove dir $d"); |
rmdir $d; |
rmdir $d; |
} |
} |
} |
} |
Line 292 sub delete_all_files {
|
Line 309 sub delete_all_files {
|
my ($files, $dir) = @_; |
my ($files, $dir) = @_; |
|
|
if (! opendir DIR, $dir) { |
if (! opendir DIR, $dir) { |
logger('ERROR', "unable to read directory $dir!\n"); |
logger('ERROR', "unable to read directory $dir!"); |
$errcnt++; |
$errcnt++; |
return; |
return; |
} |
} |
Line 306 sub delete_all_files {
|
Line 323 sub delete_all_files {
|
if (-f "$dir/$f") { |
if (-f "$dir/$f") { |
# $f is a file |
# $f is a file |
if (unlink "$dir/$f") { |
if (unlink "$dir/$f") { |
logger('INFO', "removed $f\n"); |
logger('DEBUG', "removed $f"); |
} else { |
} else { |
logger('ERROR', "unable to delete $f!\n"); |
logger('ERROR', "unable to delete $f!"); |
$errcnt++; |
$errcnt++; |
} |
} |
} elsif (-d _) { |
} elsif (-d _) { |
# $f is a directory (unlink won't work) |
# $f is a directory (unlink won't work) |
if ((system 'rm', '-r', "$dir/$f") == 0) { |
if ((system 'rm', '-r', "$dir/$f") == 0) { |
logger('INFO', "removed directory $f\n"); |
logger('DEBUG', "removed directory $f"); |
} else { |
} else { |
logger('ERROR', "unable to delete directory $f!\n"); |
logger('ERROR', "unable to delete directory $f!"); |
$errcnt++; |
$errcnt++; |
} |
} |
} else { |
} else { |
logger('ERROR', "funny object $dir/$f!\n"); |
logger('ERROR', "funny object $dir/$f!"); |
$errcnt++; |
$errcnt++; |
} |
} |
} else { |
} else { |
# $f is not in the list |
# $f is not in the list |
if (-d "$dir/$f") { |
if (-d "$dir/$f") { |
# recurse into directories |
# recurse into directories |
logger('DEBUG', "enter $dir/$f\n"); |
logger('DEBUG', "enter $dir/$f"); |
delete_all_files($files, "$dir/$f"); |
delete_all_files($files, "$dir/$f"); |
} |
} |
} |
} |
Line 339 sub delete_all_files {
|
Line 356 sub delete_all_files {
|
# main |
# main |
# |
# |
|
|
logger('START', "archiver $version at $archdate\n"); |
logger('START', "archiver $version at $archdate"); |
|
|
# make shure the right user is running this program |
# make shure the right user is running this program |
my $user = getlogin; |
my $user = getlogin; |
#if (($user ne "archive")||($user ne "root")) { |
if (($user ne "archive")&&($user ne "root")) { |
# logger("ABORT", "you ($user) must be archive or root user to run this program!"); |
logger("WARNING", "you ($user) should be archive or root user to run this program!"); |
# exit 1; |
} |
#} |
|
|
# check for .archived file |
|
if (-f "$docdir/.archived") { |
|
if (not $force_archive) { |
|
logger('ABORT', "already archived! (.archived file exists)"); |
|
exit 1; |
|
} else { |
|
logger('WARNING', "resource already archived? (.archived file exists)"); |
|
$warncnt++; |
|
} |
|
} |
|
|
# use metacheck first |
# use metacheck first |
if (system("$checkprog $docdir >/dev/null") == 0) { |
my $check_opts="-add-files"; |
logger('INFO', "resource '$docdir' check OK\n"); |
if ($rewrite_index) { |
|
$check_opts = "-replace"; |
|
} |
|
if (open CHECK, "$checkprog $check_opts $docdir |") { |
|
my @errors; |
|
my $msg; |
|
while (<CHECK>) { |
|
chomp; |
|
if (/^ERROR/) { |
|
push @errors, $_; |
|
} |
|
$msg = $_; |
|
} |
|
if ($msg =~ /^DONE/) { |
|
logger('DEBUG', "checking index file: $msg"); |
|
logger('INFO', "resource '$docdir' check OK"); |
} else { |
} else { |
logger('ABORT', "resource '$docdir' check failed!!\n"); |
logger('DEBUG', "errors checking index file:\n " . join("\n ", @errors) . "\n $msg"); |
|
logger('ABORT', "resource '$docdir' check failed!"); |
exit 1; |
exit 1; |
} |
} |
|
} else { |
|
logger('ABORT', "unable to run $checkprog"); |
|
exit 1; |
|
} |
|
# if (system("$checkprog $docdir >/dev/null") == 0) { |
|
# logger('INFO', "resource '$docdir' check OK"); |
|
# } else { |
|
# logger('ABORT', "resource '$docdir' check failed!!"); |
|
# exit 1; |
|
# } |
|
|
# read index.meta file |
# read index.meta file |
my ($document, $rootnode) = MPIWGStor::read_xml($metafile); |
my ($document, $rootnode) = MPIWGStor::read_xml($metafile); |
Line 362 my ($document, $rootnode) = MPIWGStor::r
|
Line 415 my ($document, $rootnode) = MPIWGStor::r
|
# check file and add archive date |
# check file and add archive date |
my $files_to_archive = read_resource_meta($rootnode); |
my $files_to_archive = read_resource_meta($rootnode); |
|
|
print "INFO: ", scalar keys %$files_to_archive, " files to archive\n"; |
logger('INFO', (scalar keys %$files_to_archive) . " files to archive"); |
|
|
# check for .archived file |
# remove .archived file |
if (-f "$docdir/.archived") { |
if (-f "$docdir/.archived") { |
if (unlink "$docdir/.archived") { |
if (unlink "$docdir/.archived") { |
logger('WARNING', "existing .archived file has been removed! Resource already archived?\n"); |
logger('WARNING', "existing .archived file has been removed!"); |
$warncnt++; |
$warncnt++; |
} else { |
} else { |
logger('ERROR', "unable to remove existing .archived file!\n"); |
logger('ERROR', "unable to remove existing .archived file!"); |
$errcnt++; |
$errcnt++; |
} |
} |
} |
} |
Line 382 if ($delete_junk_files) {
|
Line 435 if ($delete_junk_files) {
|
|
|
# write new index.meta |
# write new index.meta |
if ($errcnt > 0) { |
if ($errcnt > 0) { |
logger('ABORT', "there were errors!\n"); |
logger('ABORT', "there were errors!"); |
exit 1; |
exit 1; |
} else { |
} else { |
if ($fix_xml) { |
if ($fix_xml) { |
Line 394 if ($errcnt > 0) {
|
Line 447 if ($errcnt > 0) {
|
my $archived_files = run_archive(); |
my $archived_files = run_archive(); |
my $num_archfiles = scalar keys %$archived_files; |
my $num_archfiles = scalar keys %$archived_files; |
|
|
logger('INFO', "$num_archfiles files archived\n"); |
logger('INFO', "$num_archfiles files archived"); |
|
|
# check list of archived files |
# check list of archived files |
check_files($files_to_archive, $archived_files); |
check_files($files_to_archive, $archived_files); |
Line 412 if ($errcnt == 0) {
|
Line 465 if ($errcnt == 0) {
|
} |
} |
} |
} |
|
|
logger('INFO', "$warncnt warnings\n"); |
logger('INFO', "$warncnt warnings"); |
logger('INFO', "$errcnt errors\n"); |
logger('INFO', "$errcnt errors"); |
if ($errcnt > 0) { |
if ($errcnt > 0) { |
logger('ABORT', "there were errors! ($num_archfiles files archived) at " . stime(time)); |
logger('ABORT', "there were errors! ($num_archfiles files archived) at " . stime(time)); |
exit 1; |
exit 1; |