foxridge-archiver/archiver.pl - diff

Return to archiver.pl CVS log

Up to [Repository] / foxridge-archiver

Diff for /foxridge-archiver/archiver.pl between versions 1.3 and 1.7

version 1.3, 2004/07/12 15:49:17	version 1.7, 2005/10/05 13:35:12
Line 16 $\|=1;	Line 16 $\|=1;
#	#

# program version	# program version
my $version = "0.5.2 (12.7.2004)";	my $version = "0.7.2 (ROC 5.10.2005)";

	# short help
	my $help = "MPIWG archiver $version
	use: archiver [options] docpath
	options:
	-debug show debugging info
	-premigrate don't delete archived files
	-force archive even if already archived
	-replace rewrite index file
	";

	# read command line parameters
	my $args = MPIWGStor::parseargs;
	if (! scalar(%$args)) {
	print $help, "\n";
	exit 1;
	}

	# debug level
	$debug = (exists $$args{'debug'}) ? $$args{'debug'} : 0;

	# force archiving
	my $force_archive = (exists $$args{'force'}) ? $$args{'force'} : 0;

	# rewrite index file
	my $rewrite_index = (exists $$args{'replace'}) ? $$args{'replace'} : 0;

# rewrite XML file (necessary for archive date!)	# rewrite XML file (necessary for archive date!)
my $fix_xml = 1;	my $fix_xml = 1;
my $xml_changed = 0;	my $xml_changed = 0;

# XML namespace (not really implemented!)	# XML namespace (not really implemented!)
my $namespace = "";	my $namespace = "";

# archive name (archive-path element, usually == $docdir)	# archive name (archive-path element, usually == $docdir)
my $archname;	my $archname;

# archive storage date (now)	# archive storage date (now)
my $archdate = stime(time);	my $archdate = stime(time);

Line 35 my $delete_junk_files = 1;	Line 63 my $delete_junk_files = 1;
# delete data files after archiving	# delete data files after archiving
my $delete_data_files = 1;	my $delete_data_files = 1;

	# don't delete archived files with "-premigrate"
	if (exists $$args{'premigrate'}) {
	$delete_data_files = not $$args{'premigrate'};
	}
	if ($delete_data_files) {
	logger('INFO', "going to remove successfully archived files from disk");
	}


#######################################################	#######################################################
# external programs	# external programs
#	#
my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc";	my $archprog = "/opt/tivoli/tsm/client/ba/bin/dsmc";
if (! -x $archprog) {	if (! -x $archprog) {
print "ABORT: TSM client program '$archprog' missing!!\n";	logger('ABORT', "TSM client program '$archprog' missing!");
exit 1;	exit 1;
}	}
my $checkprog = "/usr/local/mpiwg/archive/metacheck";	my $checkprog = "/usr/local/mpiwg/archive/metacheck";
if (! -x $checkprog) {	if (! -x $checkprog) {
print "ABORT: meta data checking program '$checkprog' missing!!\n";	logger('ABORT', "meta data checking program '$checkprog' missing!");
exit 1;	exit 1;
}	}
# log file for archiver	# log file for archiver
my $log_file = "/var/log/mpiwg-archiver.log";	my $log_file = "/var/log/mpiwg-archiver.log";
if (! open LOG, ">>$log_file") {	if (! open LOG, ">>$log_file") {
print "ABORT: unable to write log file '$log_file'!!\n";	logger('ABORT', "unable to write log file '$log_file'!");
exit 1;	exit 1;
}	}

#######################################################	#######################################################
# check parameters that were passed to the program	# check parameters that were passed to the program
#	#
if ($#ARGV < 0) {
print "ABORT: no document directory given!\n";	my $docdir = $$args{'path'};
exit 1;
}
my $docdir = $ARGV[0];
# strip double slashes	# strip double slashes
$docdir =~ s/\/\//\//;	$docdir =~ s/\/\//\//;
# strip trailing slashes	# strip trailing slashes
$docdir =~ s/\/+$//;	$docdir =~ s/\/+$//;
if (! -d $docdir) {	if (! -d $docdir) {
print "ABORT: document directory \'$docdir\' doesn't exist!\n";	logger('ABORT', "document directory \'$docdir\' doesn't exist!");
exit 1;	exit 1;
}	}
if (($#ARGV > 0)&&($ARGV[1] eq "-premigrate")) {
$delete_data_files = 0;
}

my $metafile = "$docdir/index.meta";	my $metafile = "$docdir/index.meta";
if (! -f $metafile) {	if (! -f $metafile) {
print "ABORT: metadata index file \'$metafile\' doesn't exist!\n";	logger('ABORT', "metadata index file \'$metafile\' doesn't exist!");
exit 1;	exit 1;
}	}

Line 109 sub read_resource_meta {	Line 139 sub read_resource_meta {
# get archive-path	# get archive-path
$archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path'));	$archname = MPIWGStor::sstrip($rootnode->findvalue('child::archive-path'));
if (! $archname) {	if (! $archname) {
print "ABORT: archive-name element missing!!\n";	logger('ABORT', "archive-name element missing!");
exit 1;	exit 1;
}	}

Line 120 sub read_resource_meta {	Line 150 sub read_resource_meta {
foreach my $fn (@filenodes) {	foreach my $fn (@filenodes) {
my $name = MPIWGStor::sstrip($fn->findvalue('child::name'));	my $name = MPIWGStor::sstrip($fn->findvalue('child::name'));
my $path = MPIWGStor::sstrip($fn->findvalue('child::path'));	my $path = MPIWGStor::sstrip($fn->findvalue('child::path'));
print "FILE: ($path)$name\n";	logger('DEBUG', "FILE ($path)$name");
my $f = ($path) ? "$path/$name" : "$name";	my $f = ($path) ? "$path/$name" : "$name";
$files{$f} = $name;	$files{$f} = $name;
}	}
Line 130 sub read_resource_meta {	Line 160 sub read_resource_meta {
#	#
my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1);	my $stordatenode = ($rootnode->find('child::archive-storage-date'))->get_node(1);
if ($stordatenode) {	if ($stordatenode) {
print "WARNING: archive storage date exists! Resource already archived?\n";	logger('WARNING', "archive storage date exists! Resource already archived?");
$warncnt++;	$warncnt++;
# delete old date	# delete old date
$stordatenode->removeChildNodes;	$stordatenode->removeChildNodes;
Line 173 sub run_archive {	Line 203 sub run_archive {
}	}
}	}
} else {	} else {
print "ABORT: unable to start archive command '$archcmd'!!\n";	logger('ABORT', "unable to start archive command '$archcmd'!!");
exit 1;	exit 1;
}	}

Line 196 sub check_files {	Line 226 sub check_files {
my $fp = "$docdir/$ft";	my $fp = "$docdir/$ft";
#print " fp: $fp\n";	#print " fp: $fp\n";
if ($$archived{$fp}) {	if ($$archived{$fp}) {
print "DEBUG: $ft archived OK\n";	logger('DEBUG', "$ft archived OK");
$$archived{$fp} = "OK";	$$archived{$fp} = "OK";
} else {	} else {
print "ERROR: file '$ft' missing from archive!\n";	logger('ERROR', "file '$ft' missing from archive!");
$errcnt++;	$errcnt++;
}	}
}	}
Line 208 sub check_files {	Line 238 sub check_files {
if ($$archived{$fa} ne "OK") {	if ($$archived{$fa} ne "OK") {
my ($fn, $fp) = MPIWGStor::split_file_path($fa);	my ($fn, $fp) = MPIWGStor::split_file_path($fa);
if ($MPIWGStor::index_files{$fn}) {	if ($MPIWGStor::index_files{$fn}) {
print "DEBUG: $fa ignored\n";	logger('DEBUG', "$fa ignored");
$na--;	$na--;
} else {	} else {
print "WARNING: $fa archived but not in list!\n";	logger('WARNING', "$fa archived but not in list!");
$warncnt++;	$warncnt++;
}	}
}	}
}	}

if ($nt > $na) {	if ($nt > $na) {
print "WARNING: less files were archived ($na vs. $nt)!\n";	logger('WARNING', "less files were archived ($na vs. $nt)!");
} elsif ($na > $nt) {	} elsif ($na > $nt) {
print "WARNING: more files were archived ($na vs. $nt)!\n";	logger('WARNING', "more files were archived ($na vs. $nt)!");
}	}

}	}
Line 248 sub delete_files {	Line 278 sub delete_files {
next unless (-f $f);	next unless (-f $f);
# delete files	# delete files
if (unlink $f) {	if (unlink $f) {
print "INFO: remove $f ($fn)\n";	logger('DEBUG', "remove $f ($fn)");
} else {	} else {
print "ERROR: unable to delete $f!\n";	logger('ERROR', "unable to delete $f!");
$errcnt++;	$errcnt++;
}	}
}	}
Line 262 sub delete_files {	Line 292 sub delete_files {
# dont't remove document dir (shouldn't be empty anyway)	# dont't remove document dir (shouldn't be empty anyway)
next if ($d eq $docdir);	next if ($d eq $docdir);
if (-d $d) {	if (-d $d) {
print "INFO: remove dir $d\n";	logger('DEBUG', "remove dir $d");
rmdir $d;	rmdir $d;
}	}
}	}
Line 279 sub delete_all_files {	Line 309 sub delete_all_files {
my ($files, $dir) = @_;	my ($files, $dir) = @_;

if (! opendir DIR, $dir) {	if (! opendir DIR, $dir) {
print "ERROR: unable to read directory $dir!\n";	logger('ERROR', "unable to read directory $dir!");
$errcnt++;	$errcnt++;
return;	return;
}	}
Line 293 sub delete_all_files {	Line 323 sub delete_all_files {
if (-f "$dir/$f") {	if (-f "$dir/$f") {
# $f is a file	# $f is a file
if (unlink "$dir/$f") {	if (unlink "$dir/$f") {
print "INFO: removed $f\n";	logger('DEBUG', "removed $f");
} else {	} else {
print "ERROR: unable to delete $f!\n";	logger('ERROR', "unable to delete $f!");
$errcnt++;	$errcnt++;
}	}
} elsif (-d _) {	} elsif (-d _) {
# $f is a directory (unlink won't work)	# $f is a directory (unlink won't work)
if ((system 'rm', '-r', "$dir/$f") == 0) {	if ((system 'rm', '-r', "$dir/$f") == 0) {
print "INFO: removed directory $f\n";	logger('DEBUG', "removed directory $f");
} else {	} else {
print "ERROR: unable to delete directory $f!\n";	logger('ERROR', "unable to delete directory $f!");
$errcnt++;	$errcnt++;
}	}
} else {	} else {
print "ERROR: funny object $dir/$f!\n";	logger('ERROR', "funny object $dir/$f!");
$errcnt++;	$errcnt++;
}	}
} else {	} else {
# $f is not in the list	# $f is not in the list
if (-d "$dir/$f") {	if (-d "$dir/$f") {
# recurse into directories	# recurse into directories
print "DEBUG: enter $dir/$f\n";	logger('DEBUG', "enter $dir/$f");
delete_all_files($files, "$dir/$f");	delete_all_files($files, "$dir/$f");
}	}
}	}
Line 326 sub delete_all_files {	Line 356 sub delete_all_files {
# main	# main
#	#

print "START: archiver $version at $archdate\n";	logger('START', "archiver $version at $archdate");

# make shure the right user is running this program	# make shure the right user is running this program
my $user = getlogin;	my $user = getlogin;
#if (($user ne "archive")\|\|($user ne "root")) {	if (($user ne "archive")&&($user ne "root")) {
# logger("ABORT", "you ($user) must be archive or root user to run this program!");	logger("WARNING", "you ($user) should be archive or root user to run this program!");
# exit 1;	}
#}
	# check for .archived file
	if (-f "$docdir/.archived") {
	if (not $force_archive) {
	logger('ABORT', "already archived! (.archived file exists)");
	exit 1;
	} else {
	logger('WARNING', "resource already archived? (.archived file exists)");
	$warncnt++;
	}
	}

# use metacheck first	# use metacheck first
if (system("$checkprog $docdir >/dev/null") == 0) {	my $check_opts="-add-files";
print "INFO: resource '$docdir' check OK\n";	if ($rewrite_index) {
	$check_opts = "-replace";
	}
	if (open CHECK, "$checkprog $check_opts $docdir \|") {
	my @errors;
	my $msg;
	while (<CHECK>) {
	chomp;
	if (/^ERROR/) {
	push @errors, $_;
	}
	$msg = $_;
	}
	if ($msg =~ /^DONE/) {
	logger('DEBUG', "checking index file: $msg");
	logger('INFO', "resource '$docdir' check OK");
} else {	} else {
print "ABORT: resource '$docdir' check failed!!\n";	logger('DEBUG', "errors checking index file:\n " . join("\n ", @errors) . "\n $msg");
	logger('ABORT', "resource '$docdir' check failed!");
exit 1;	exit 1;
}	}
	} else {
	logger('ABORT', "unable to run $checkprog");
	exit 1;
	}
	# if (system("$checkprog $docdir >/dev/null") == 0) {
	# logger('INFO', "resource '$docdir' check OK");
	# } else {
	# logger('ABORT', "resource '$docdir' check failed!!");
	# exit 1;
	# }

# read index.meta file	# read index.meta file
my ($document, $rootnode) = MPIWGStor::read_xml($metafile);	my ($document, $rootnode) = MPIWGStor::read_xml($metafile);
Line 349 my ($document, $rootnode) = MPIWGStor::r	Line 415 my ($document, $rootnode) = MPIWGStor::r
# check file and add archive date	# check file and add archive date
my $files_to_archive = read_resource_meta($rootnode);	my $files_to_archive = read_resource_meta($rootnode);

print "INFO: ", scalar keys %$files_to_archive, " files to archive\n";	logger('INFO', (scalar keys %$files_to_archive) . " files to archive");

# check for .archived file	# remove .archived file
if (-f "$docdir/.archived") {	if (-f "$docdir/.archived") {
if (unlink "$docdir/.archived") {	if (unlink "$docdir/.archived") {
print "WARNING: existing .archived file has been removed! Resource already archived?\n";	logger('WARNING', "existing .archived file has been removed!");
$warncnt++;	$warncnt++;
} else {	} else {
print "ERROR: unable to remove existing .archived file!\n";	logger('ERROR', "unable to remove existing .archived file!");
$errcnt++;	$errcnt++;
}	}
}	}
Line 369 if ($delete_junk_files) {	Line 435 if ($delete_junk_files) {

# write new index.meta	# write new index.meta
if ($errcnt > 0) {	if ($errcnt > 0) {
print "ABORT: there were errors!\n";	logger('ABORT', "there were errors!");
exit 1;	exit 1;
} else {	} else {
if ($fix_xml) {	if ($fix_xml) {
Line 381 if ($errcnt > 0) {	Line 447 if ($errcnt > 0) {
my $archived_files = run_archive();	my $archived_files = run_archive();
my $num_archfiles = scalar keys %$archived_files;	my $num_archfiles = scalar keys %$archived_files;

print "INFO: $num_archfiles files archived\n";	logger('INFO', "$num_archfiles files archived");

# check list of archived files	# check list of archived files
check_files($files_to_archive, $archived_files);	check_files($files_to_archive, $archived_files);
Line 399 if ($errcnt == 0) {	Line 465 if ($errcnt == 0) {
}	}
}	}

print "INFO: $warncnt warnings\n";	logger('INFO', "$warncnt warnings");
print "INFO: $errcnt errors\n";	logger('INFO', "$errcnt errors");
if ($errcnt > 0) {	if ($errcnt > 0) {
print "ABORT: there were errors! ($num_archfiles files archived) at ", stime(time), "\n";	logger('ABORT', "there were errors! ($num_archfiles files archived) at " . stime(time));
exit 1;	exit 1;
} else {	} else {
print "DONE: $num_archfiles files archived at ", stime(time), "\n";	logger('DONE', "$num_archfiles files archived at " . stime(time));
}	}

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.3
changed lines
	Added in v.1.7