WO3_Euclid_1966: Euclid_1966.pl

#!/usr/bin/perl -w
use strict;
use warnings;
use utf8;
binmode STDOUT, ':utf8';
use integer;


# Euclid_1966.pl
#
# author: Wolfgang Schmidle
# (c) Max Planck Institute for the History of Science, Berlin, Germany
# version 1.3, 2009-04-07
#
# This script processes the raw text of Euclid (1966), i.e. it turns (a local copy of)
# http://pythia.mpiwg-berlin.mpg.de/department1/mpdl/raw-texts/WO3_Euclid_1966.txt/WO3_Euclid_1966_V1.txt
# into a valid XML file.
#
# For Euclid (1966) see also
# https://itgroup.mpiwg-berlin.mpg.de:8080/tracs/mpdl-project-content/wiki/WO3_Euclid_1966
#
# Usage:  perl Euclid_1966.txt > Euclid_1966.xml


# Some remaining issues:
#
# - Replacing <001> by the correct character works fine, but <002> is in a higher plane of Unicode and kills off oXygen, 
#   so I have used the simpler standard version of this character.
# - Some <?> have been post-processed already, and I have removed the respective <?> tags because a <?> tag 
#   has no value in itself once the line has been checked. I have compiled a list of lines which contain <?> and/or @ 
#   and haven't been post-processed yet. 
# 
# - The <desc> and <var> in figures have not been used very consistently by Formax. (They did not use <cap> 
#   at all, but I only know of one figure where it would make sense to use it.)
# - Figures and note have no place attribute.
# - No <num>, <var> (outside of figures), <ptr>, no IDs yet.
#
# - The parts of problems with more than one part have not been encoded yet.
# - Four books, i.e. four titles, one in the front (attribute n=1) and three in the body (n=1, 2, 3).


my $rawtext = shift @ARGV;


# chinese into arabic numbers

my $digit = "[一二三四五六七八九]";
my %arabic = ( 一 => 1, 二 => 2, 三 => 3, 四 => 4, 五 => 5, 六 => 6, 七 => 7, 八 => 8, 九 => 9 );  

sub chinese2arabic {
        my $ch = shift;
        if ($ch =~ m!($digit)十($digit)!) { return 10*$arabic{$1}+$arabic{$2}; } 
        if ($ch =~ m!($digit)十!) { return 10*$arabic{$1}; } 
        if ($ch =~ m!十($digit)!) { return 10+$arabic{$1}; } 
        if ($ch =~ m!十!) { return 10; }
        if ($ch =~ m!($digit)!) { return $arabic{$1}; }
        return "???";
}

my $moderndigit = "[一二三四五六七八九〇]";
my %modernarabic = ( 一 => 1, 二 => 2, 三 => 3, 四 => 4, 五 => 5, 六 => 6, 七 => 7, 八 => 8, 九 => 9, 〇 => 0 );  

sub modernchinese2arabic {
        my $ch = shift;
        my $result = 0;
        while ($ch ne "") { $result = 10 * $result + $modernarabic{substr($ch, 0, 1, "")}; }
        return $result;
}


# text-specific preparations 

my $number = "$digit|$digit?十$digit?";  # numbers from 1 to 99 (there is no higher number in any heading)
my $nth = "第";

my $headingType = "[界|求|論|題]"; 
my %englishHeadingType = ( 界 => "definition", 求 => "construction", 論 => "axiom", 題 => "problem" );


# global variables

my @text;

# counters
my $line;
my $pb = 0;
my %div = ( 界 => 0, 求 => 0, 論 => 0, 題 => 0); 
my $note = 0;

# booleans
my $inParagraph = 0;
my $lineStartsWithS = 0;
my $inDiv = 0;
my $inFig = 0;
my $inTb = 0;


# read in the raw text

open (RAWTEXT, "<:utf8", $rawtext) or die "Can't open the raw text file \"$rawtext\"!\n";
while (<RAWTEXT>) {

        s!\r\n!!;  # remove Windows line ends
        push @text, $_;
}
close (RAWTEXT);


# emendations of the raw text
# (see also the reduplicated ECHO pages in the processing of page breaks)

$line = 0;

for (@text) {

        $line++;

        # normalize the zero in modern page numbers
        s!○!〇!g;  # white circle U+25CB --> ideographic number zero U+3007
        
        # ignore outdentation in the preface
        if ($line < 153) { s!<p x>!<p>!; } 

        # pre-process notes that continue on the next line      
        if ($text[$line-1] =~ m!</sm>$!) {
                my $counterpart = $line;
                if ($text[$counterpart] =~ m!^<pb!) { $counterpart++; }
                if ($text[$counterpart] =~ m!^<sm>!) {  
                        $text[$line-1] =~ s!</sm>$!!;
                        $text[$counterpart] =~ s!^<sm>!!;
                }
        }
        
        # fill in the unknown characters (MSi)
        s!<001>!轂!g;  # s!<001>!<unknown code="001" unicode="8F42">轂</unknown>!g;
        s!<002>!<unknown code="002" unicode="2F88D">庶</unknown>!g; # the actual Unicode character 庶 breaks oXygen

        # clarify <?> (the list is not complete!)
        s!<？>!<?>!; # line 811: fullwidth question mark U+FF1F --> ASCII question mark U+003F
        s!餘<\?>二倣此。!餘二倣此。!; # line 811
        s!愈<\?>!愈!g; # MSi: the reading is correct
        s!。<\?>!。!;  # lines 1041, 1087, 2104, 3855: all unclear periods are plausible
        # (e.g. line 1041: MSi: It is in the middle of a sentence, but a period at this position is quite common nonetheless.) 
        s!有兩種幾何<\?>。!有兩種幾何。!; # line 2208
        s!其元<\?>大者!其元大者!; # line 2208 again

        # replace the @
        s!\@增至於無窮。!遞增至於無窮。!; # line 2208 again: U+905E
        s!\@減至於無窮。!遞減至於無窮。!; # line 2208 again: U+905E
        s!<p>@!<p><gap/>!; # lines 3642, 3667, 3815: missing lines of text (last lines on pages 278, 280 and 288)

        # missing line breaks (the list is not complete!)
        s!小於兩直角。則此二橫直線。!小於兩直角。則此二橫<lb/>直線。!; # line 403; may have to do with the neighboring figure
        s!俱小於直角。或幷之小於兩直角。!俱小於直角。或幷之小<lb/>於兩直角。!; # line 404
        
        # normalize the hash in the table
        s!＃!#!g; # fullwidth number sign U+FF03 --> ASCII hash, i.e. number sign U+0023
        
        # move the only table in the text (ECHO p.327) out of its surrounding sentence
        s!却云十六與十二之比例。若!却云十六與十二之比例。!;  # line 4562
        s!八與三、及二與四之比例。!若<lb/>八與三、及二與四之比例。!; #line 4573

        # misc. emendations
        s!N12<114608657010!N12x114608657010!; # line 5: replace "<" in library stamp junk
        s!<pb 六><h>幾何原本　卷一之首</h>!<pb 六><rh>幾何原本　卷一之首</rh>!;  # line 245 (obvious mistake)
        s!<h>後支前己正論</h>!<p>後支前己正論</p>!;  # line 2175 (Tian Miao: wrong tag)
        if ($line == 2992) { s!<h>第三十四題</h>!<h>第十四題</h>!; }  # line 2992 (obvious mistake)

}


# process the raw text

# The metadata add 31 lines to the text. Apart from these prefix lines, the line structure of the original is preserved.

    print <<'METADATA';
<?xml version="1.0" encoding="UTF-8"?>
<echo xmlns="http://www.mpiwg-berlin.mpg.de/ns/echo/1.0/"
      xmlns:dc="http://purl.org/dc/elements/1.1/"
      xmlns:dcq="http://purl.org/dc/qualifiers/1.0/"
      xmlns:dct="http://purl.org/dc/terms/1.0/"
      xmlns:echo="http://www.mpiwg-berlin.mpg.de/ns/echo/1.0/"
      xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
      xmlns:xhtml="http://www.w3.org/1999/xhtml"
      xmlns:xlink="http://www.w3.org/1999/xlink"
      xmlns:xml="http://www.w3.org/XML/1998/namespace">
  <metadata>
    <dc:creator>Euclid</dc:creator>
    <dc:title xml:lang="zh">Jihe yuanben</dc:title>
    <dc:title xml:lang="zh">幾何原本</dc:title>
    <dc:date>
      <rdf:Description>
        <dcq:dateScheme>ISO 8601</dcq:dateScheme>
        <rdf:value>1966</rdf:value>
      </rdf:Description>
    </dc:date>
    <dc:language>
      <rdf:Description>
        <dcq:languageScheme>ISO 639-2</dcq:languageScheme>
        <rdf:value>zho-Hant</rdf:value>
      </rdf:Description>
    </dc:language>
    <dc:identifier>test.xml</dc:identifier>
    <dc:rights>open access</dc:rights>
    <dct:license>http://echo.mpiwg-berlin.mpg.de/policy/oa_basics/declaration</dct:license>
  </metadata>
   <text xml:lang="zh">
METADATA


$line = 0;

for (@text) {

        $line++;
        
        # unknown characters
        # (must be before figures so that of <?> in a <var> will be processed)
        s!<\?>!<unsure/>!g;

        
        # figures
        # (must be before the tagging of div's to avoid </div></fig>)
        # (the internal ordering is important for successive figures)
        
        if ($inFig) {
                if (m!<cap>! || m!<desc>! || m!<var>!) { 
                        print "$_\n";
                        next;
                }
                print "</figure>";
                $inFig = 0;
        }
        if (s!<fig>!<figure>!) {   #    !<figure place="text">! 
                $inFig = 1; 
                print "$_\n";
                next;
        }


        # the table on ECHO p.327
        # (must be before paragraphs)
        
        if (m!<tb>!) { $inTb = 1; }

        if ($inTb) {
                unless (s!<(/?)tb>!<$1xhtml:table>!) {
                        $_ = "<xhtml:tr><xhtml:td>$_</xhtml:td></xhtml:tr>";
                        s!#!</xhtml:td><xhtml:td>!g;
                }
                print "$_\n";
                if (m!</xhtml:table>!) { $inTb = 0; }
                next;
        }


        # ad hoc tagging of book covers, preface, chapters, chapter heads, chapter mains, backmatter
        # including some additional lowest-level closing div's
        
        sub beginDiv {
                $inDiv = 0;
                foreach my $key (keys %div) { $div{$key} = 0; } 
        }
        
        if ( $line == 1) { print '<front><div level="1" type="title" n="1">'; }  # 1 - 32
        if ( $line == 33) { print '</div>'; }

        if ( $line == 33) { print '<div level="1" type="preface" n="1">'; } # 33 - 152
        if ( $line == 153) { print '</div></front>'; }

        if ( $line == 153) { print '<body><div level="1" type="chapter" n="1"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
        if ( $line == 452) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
        if ( $line == 1364) { print '</div></div></div>'; }

        if ( $line == 1364) {print '<div type="title" n="1">'; } # 1364 - 1385
        if ( $line == 1386) { print '</div>'; }

        if ( $line == 1386) { print '<div level="1" type="chapter" n="2"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
        if ( $line == 1416) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
        if ( $line == 1753) { print '</div></div></div>'; }

        if ( $line == 1753) { print '<div level="1" type="chapter" n="3"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
        if ( $line == 1848) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
        if ( $line == 2722) { print '</div></div></div>'; }

        if ( $line == 2722) { print '<div type="title" n="2">'; } # 2722 - 2744
        if ( $line == 2745) { print '</div>'; }

        if ( $line == 2745) { print '<div level="1" type="chapter" n="4"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
        if ( $line == 2787) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
        if ( $line == 3109) { print '</div></div></div>'; }

        if ( $line == 3109) { print '<div level="1" type="chapter" n="5"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
        if ( $line == 3520) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
        if ( $line == 4387) { print '</div></div></div>'; }

        if ( $line == 4387) { print '<div type="title" n="3">'; } # 4387 - 4392
        if ( $line == 4393) { print '</div>'; }

        if ( $line == 4393) { print '<div level="1" type="chapter" n="6"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
        if ( $line == 4594) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
        if ( $line == 5984) { print '</div></div></div></body>'; }

        if ( $line == 5984) { print '<back><div type="backmatter">'; } # 5984 - end; </div></back> is printed together with </xml>, see below.


        # page breaks

        if (m!<pb!) {
                $pb++;
                
#               # The ECHO pages 215 to 220 reduplicate the pages 209 to 214 and have been typed only once by Formax
#               if ($pb == 215) { $pb = 221; } 
                
                m!^<pb( ($moderndigit+))?>(<rh>(.+)</rh>)? *$!;
                my $pagenumber = ($2 ? " o=\"$2\" oE=\"" . &modernchinese2arabic($2). "\"" : "");
                my $runninghead = ($4 ? " rhead=\"$4\"" : "" );
                my $link = 'http://echo.mpiwg-berlin.mpg.de/ECHOdocuView/ECHOzogiLib?'.
                        'url=/mpiwg/online/permanent/library/02NT95YF/pageimg&amp;mode=imagepath&amp;pn='.$pb;
                $_ = "<pb$pagenumber$runninghead n=\"$pb\" xlink:href='$link'/>";
                print "$_\n";
                next;
        }

        
        # headings

        if (m!<h>!) { 

                s!<h>!<head>!;
                s!</h>!</head>!;

                # notes in headings
                s!　?\\\\　?!<lb type="halfline"/>!g;
                s!<sm>(.+)</sm>!<note>$1</note>!;

                # headings at the lowest level
                if (m!<head>$nth($number)($headingType).*?</head>$!)  { 
                        my $o = $1;
                        my $oE = &chinese2arabic($o);
                        my $type = $2;
                        my $typeE = $englishHeadingType{$2};
                        s!　(.+?)</head>!<note>$1</note></head>!;                
                        $div{$type}++;
                        if ($div{$type} != $oE) { die "$line: The numbering of the div's is incorrect!\n"; }
                        
                        if ($inDiv) { print "</div>"; }
                        print '<div type="'.$type.'" typeE="'.$typeE.'" o="'.$o.'" level="3" n="'.$div{$type}.'">'; 
                        $inDiv = 1;
                } 
                
                # all headings
                print "$_\n";
                next;
        }


        # paragraphs
        # (<p> is always at the beginning, </p> always at the end of a line)
        # inside a paragraph: <pb ...>, <rh>; <sm>; <fig>, <desc>, <var>; 1x <tb>; 3x <002> 
        
        my $outdented = s!<p x>!<p>!;

        if (m!<p>!) { $inParagraph = 1; }
        
        if ($inParagraph) {

                # normalize the periods: there should always be a period before </p> and </sm>
                # (if the period is missing, insert an ASCII period)
                s!([^。])</p>!$1\.</p>!g;
                s!</sm>\.</p>!</sm></p>!;
                s!([^。])<sm>!$1\.<sm>!g;
                s!^\.<sm>!<sm>!;
                s!<p>\.<sm>!<p><sm>!;
                s!([^。])</sm>!$1\.</sm>!g;
        
                # turn small text into notes
                s!<sm>!<note>!g;  # <sm> and </sm> need not be in the same line
                s!</sm>!</note>!g;
                s!　?\\\\　?!<lb type="halfline"/>!g;

                # tag sentences
                s!<p>!<p><s>!;
                if ($lineStartsWithS) { $_ = "<s>$_"; $lineStartsWithS = 0; } 
                s![。.]!$&</s><s>!g;
                if (s!<s>$!!) { $lineStartsWithS = 1; }
                s!<s><note>!<note><s>!g;      #  ???
                s!<s></note>!</note><s>!g;
                s!<s><lb type="halfline"/>!<lb type="halfline"/><s>!g;
                s!<s></p>!</p>!;
                
                # end each line with <lb/>
                $_ .= "<lb/>" unless (($line < $#text) &&(($text[$line] =~ m!<tb>!) || ($text[$line] =~ m!<pb!)));
                s!</p><lb/>!</p>!;
                s!<s><lb/>!<lb/><s>!g;
                
                # outdented paragraphs
                if ($outdented) { s!<p>!<p outdented="1">!; }
                
                print "$_\n";

                if (m!</p>!) { $inParagraph = 0; }
                next;
        }
        

        # everything else
        
        print "$_\n";   
}


print "</div>";  # end of backmatter
print "</back></text></echo>\n";