Annotation of foxridge-archiver/harvestmeta.pl, revision 1.1
1.1 ! casties 1: #!/usr/local/bin/perl -w
! 2:
! 3: use strict;
! 4: use XML::SAX;
! 5: use DBI;
! 6:
! 7: use lib '/usr/local/mpiwg/archive';
! 8: use MPIWGStor;
! 9: use HarvestmetaHandler;
! 10:
! 11: # make output unbuffered
! 12: $|=1;
! 13:
! 14: #######################################################
! 15: # internal parameters
! 16: #
! 17:
! 18: # program version
! 19: my $version = "0.1 (08.06.2004)";
! 20:
! 21: # read command line parameters
! 22: my $args = MPIWGStor::parseargs;
! 23:
! 24: # debug level
! 25: $debug = (exists $$args{'debug'}) ? $$args{'debug'} : 0;
! 26:
! 27: # XML namespace (not really implemented!)
! 28: my $namespace = "";
! 29:
! 30: # delete and rebuild database
! 31: my $purgeDB = (exists $$args{'purgedb'});
! 32:
! 33: # database connection
! 34: my $dbh = DBI->connect("dbi:Pg:dbname=storage", "archiver", "");
! 35: if (! $dbh) {
! 36: logger('ABORT', "unable to connect to database!");
! 37: exit 1;
! 38: }
! 39: $dbh->{AutoCommit} = 0;
! 40: my $dbNextFileId;
! 41: my $dbNewFile;
! 42: my $dbNewMeta;
! 43: my $dbClearMeta;
! 44: my $dbFindFileName;
! 45: my $dbFindFilePath;
! 46: my $dbClearFile;
! 47: my $dbFindFileFlag;
! 48: my $dbFindFileFlagPath;
! 49: my $dbSetFileFlag;
! 50: my $dbClearAllFileFlag;
! 51:
! 52: #######################################################
! 53: # check parameters that were passed to the program
! 54: #
! 55: my $basedir = $$args{'path'};
! 56: if (! $basedir) {
! 57: logger("ABORT", "no document directory given!");
! 58: exit 1;
! 59: }
! 60: # strip trailing slashes
! 61: $basedir =~ s/\/$//;
! 62: if (! -d $basedir) {
! 63: logger("ABORT", "document directory \'$basedir\' doesn't exist!");
! 64: exit 1;
! 65: }
! 66:
! 67: my $metaParserHandler = HarvestmetaHandler->new;
! 68: my $metaParser = XML::SAX::ParserFactory->parser(Handler => $metaParserHandler);
! 69:
! 70: #######################################################
! 71: # internal variables
! 72: #
! 73:
! 74: # number of errors
! 75: my $errcnt = 0;
! 76: # number of warnings
! 77: my $warncnt = 0;
! 78:
! 79: # number of files on fs
! 80: my $fcnt = 0;
! 81: # number of index files
! 82: my $idxcnt = 0;
! 83:
! 84: #######################################################
! 85: # subroutines
! 86: #
! 87:
! 88: #
! 89: # readAllFiles($realdir, $basedir, \%files, \%dirs)
! 90: #
! 91: # reads all files and directories below $realdir and puts the
! 92: # files in %files and directories in %dirs
! 93: # $basedir is only for recursion, it should be empty when called
! 94: # from outside
! 95: #
! 96: sub readAllFiles {
! 97: my ($directory, $basedir) = @_;
! 98: my $cnt = 0;
! 99:
! 100: if (! opendir DIR, $directory) {
! 101: return 0;
! 102: }
! 103: my @dirfiles = readdir DIR;
! 104: foreach my $fn (@dirfiles) {
! 105: # ignore names starting with a dot
! 106: next if ($fn =~ /^\./);
! 107: # ignore other silly files
! 108: next if ($junk_files{$fn});
! 109:
! 110: $cnt++;
! 111: $fcnt++;
! 112: my $f = "$directory/$fn";
! 113: my $docf = ($basedir) ? "$basedir/$fn" : $fn;
! 114: #logger('DEBUG', "fs_file: \"$f\"");
! 115: if (-f $f) {
! 116: #logger(" is file");
! 117: if ($fn eq "index.meta") {
! 118: harvestFile($fn, $directory);
! 119: }
! 120: } elsif (-d _) {
! 121: #logger(" is dir");
! 122: # recurse into directory
! 123: $cnt += readAllFiles($f, $docf);
! 124: }
! 125: }
! 126: return $cnt;
! 127: }
! 128:
! 129: #
! 130: # cleanUnmarkedFiles($basepath)
! 131: #
! 132: # deletes all unflagged file and meta entries.
! 133: #
! 134: sub cleanUnmarkedFiles {
! 135: my ($basepath) = @_;
! 136: my $rv = $dbFindFileFlagPath->execute("${basepath}%");
! 137: my $ids = $dbFindFileFlagPath->fetchall_arrayref;
! 138: for my $i (@$ids) {
! 139: my $id = $$i[0];
! 140: logger('DEBUG', "cleaning file and meta of id: $id");
! 141: $dbClearMeta->execute($id);
! 142: $dbClearFile->execute($id);
! 143: $dbh->commit;
! 144: }
! 145: }
! 146:
! 147: #
! 148: # harvestFile($filename, $filepath)
! 149: #
! 150: # reads the index file $filename at $filepath and puts the contents
! 151: # in the database.
! 152: #
! 153: sub harvestFile {
! 154: my ($filename, $filepath) = @_;
! 155: logger('DEBUG', "looking at file '$filename' at '$filepath'");
! 156: # get file time
! 157: my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
! 158: $atime,$mtime,$ctime,$blksize,$blocks)
! 159: = stat("$filepath/$filename");
! 160: my $filetime = stime($mtime);
! 161: # register file in db
! 162: my $fid = registerFile("$filepath/$filename", $filetime);
! 163: if ($fid) {
! 164: # file is new/modified
! 165: # parse index file
! 166: $metaParser->parse_uri("$filepath/$filename");
! 167: my @data = $metaParserHandler->getData();
! 168: logger('DEBUG', "parsed $#data+1 elements");
! 169: registerMeta($fid, @data);
! 170: }
! 171: $idxcnt++;
! 172: logger('INFO', "$idxcnt index files of $fcnt") if ($idxcnt % 10 == 0) ;
! 173: }
! 174:
! 175: #
! 176: # $fileid = registerFile($filepath, $filetime)
! 177: #
! 178: # returns the file ID for the file $filepath. If necessary it
! 179: # will be added to the database. returns 0 if an update is not necessary.
! 180: #
! 181: sub registerFile {
! 182: my ($filepath, $filetime) = @_;
! 183: my $fileid = 0;
! 184: # look if file is in db
! 185: my $rv = $dbFindFileName->execute($filepath);
! 186: my $mtime;
! 187: ($fileid, $mtime) = $dbFindFileName->fetchrow_array;
! 188: if ($fileid) {
! 189: # file is in db
! 190: # update flag
! 191: $dbSetFileFlag->execute($fileid, 1);
! 192: $dbh->commit;
! 193: my $stime = s2stime($mtime);
! 194: if ($stime ge $filetime) {
! 195: # if its current return 0
! 196: logger('DEBUG', "file: $fileid is old! time: '$stime' (vs '$filetime')");
! 197: return 0;
! 198: } else {
! 199: logger('DEBUG', "file: $fileid is new! time: '$stime' (vs '$filetime')");
! 200: }
! 201: }
! 202: if (! $fileid) {
! 203: # get a new file id
! 204: my $rv = $dbNextFileId->execute;
! 205: ($fileid) = $dbNextFileId->fetchrow_array;
! 206: logger('DEBUG', "DB newfile: id=$fileid filename=$filepath mtime=$filetime");
! 207: $dbNewFile->execute($fileid, $filepath, $filetime);
! 208: # update flag
! 209: $dbSetFileFlag->execute($fileid, 1);
! 210: $dbh->commit;
! 211: }
! 212: return $fileid;
! 213: }
! 214:
! 215: #
! 216: # registerMeta($fileid, @meta)
! 217: #
! 218: # adds the metadata information @meta for $fileid to the database.
! 219: #
! 220: sub registerMeta {
! 221: my ($fileid, @meta) = @_;
! 222: logger('DEBUG', "DB newmeta: fileid=$fileid ($#meta)");
! 223: my $idx = 0;
! 224: foreach my $keyval (@meta) {
! 225: #logger('DEBUG', " DB meta: $$keyval[0]=$$keyval[1]");
! 226: $dbNewMeta->execute($fileid, $idx++, $$keyval[0], $$keyval[2], $$keyval[1]);
! 227: }
! 228: $dbh->commit;
! 229: logger('INFO', "added $idx elements (file $fileid)");
! 230: }
! 231:
! 232: #
! 233: # initdb()
! 234: #
! 235: # initialises the database connection.
! 236: #
! 237: sub initDB {
! 238: my $rv;
! 239: # clean tables
! 240: if ($purgeDB) {
! 241: $rv = $dbh->do("delete from files");
! 242: $rv = $dbh->do("delete from meta");
! 243: if ($dbh->err) {
! 244: logger('ABORT', "unable to clean table!");
! 245: exit 1;
! 246: }
! 247: $dbh->commit;
! 248: }
! 249:
! 250: # clear flags
! 251: $rv = $dbh->do("create temporary table file_flags ( fileid integer primary key, flag integer )");
! 252: $dbh->commit;
! 253:
! 254: # prepare statements
! 255: $dbNextFileId = $dbh->prepare("select nextval('files_id_seq')");
! 256: $dbNewFile = $dbh->prepare("insert into files (id, filename, mtime) values (?,?,?)");
! 257: $dbFindFileName = $dbh->prepare("select id,mtime from files where filename=?");
! 258: $dbFindFilePath = $dbh->prepare("select id,filename,flag from files where filename like ?");
! 259: $dbClearFile = $dbh->prepare("delete from files where id=?");
! 260: $dbFindFileFlag = $dbh->prepare("select fileid from file_flags where flag=?");
! 261: $dbFindFileFlagPath = $dbh->prepare("select id from files left outer join file_flags on files.id=file_flags.fileid where filename like ? and flag is null");
! 262: $dbSetFileFlag = $dbh->prepare("insert into file_flags (fileid, flag) values (?,?)");
! 263: $dbNewMeta = $dbh->prepare("insert into meta (fileid, idx, tags, attributes, content) values (?,?,?,?,?)");
! 264: $dbClearMeta = $dbh->prepare("delete from meta where fileid=?");
! 265:
! 266: }
! 267:
! 268: #######################################################
! 269: # main
! 270: #
! 271:
! 272: logger("INFO", "harvestmeta $version");
! 273:
! 274: initDB();
! 275:
! 276: # read and process all files under $basedir
! 277: my $fnum = readAllFiles($basedir, "");
! 278: # delete orphaned data (under $basedir)
! 279: cleanUnmarkedFiles($basedir);
! 280:
! 281: logger("INFO", "analysed $idxcnt of $fnum files!");
! 282: logger("INFO", "$warncnt warnings");
! 283: logger("INFO", "$errcnt errors");
! 284: if ($errcnt > 0) {
! 285: logger("ABORT", "there were errors!");
! 286: exit 1;
! 287: } else {
! 288: logger("DONE", "all index files read successfully!");
! 289: }
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to harvestmeta.pl CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / foxridge-archiver