﻿#!/usr/bin/perl -w
use strict;
use warnings;
use utf8;
binmode STDOUT, ':utf8';
use integer;


# Euclid_1966.pl
#
# author: Wolfgang Schmidle
# (c) Max Planck Institute for the History of Science, Berlin, Germany
# version 1.3, 2009-04-07
#
# This script processes the raw text of Euclid (1966), i.e. it turns (a local copy of)
# http://pythia.mpiwg-berlin.mpg.de/department1/mpdl/raw-texts/WO3_Euclid_1966.txt/WO3_Euclid_1966_V1.txt
# into a valid XML file.
#
# For Euclid (1966) see also
# https://itgroup.mpiwg-berlin.mpg.de:8080/tracs/mpdl-project-content/wiki/WO3_Euclid_1966
#
# Usage:  perl Euclid_1966.txt > Euclid_1966.xml


# Some remaining issues:
#
# - Replacing <001> by the correct character works fine, but <002> is in a higher plane of Unicode and kills off oXygen, 
#   so I have used the simpler standard version of this character.
# - Some <?> have been post-processed already, and I have removed the respective <?> tags because a <?> tag 
#   has no value in itself once the line has been checked. I have compiled a list of lines which contain <?> and/or @ 
#   and haven't been post-processed yet. 
# 
# - The <desc> and <var> in figures have not been used very consistently by Formax. (They did not use <cap> 
#   at all, but I only know of one figure where it would make sense to use it.)
# - Figures and note have no place attribute.
# - No <num>, <var> (outside of figures), <ptr>, no IDs yet.
#
# - The parts of problems with more than one part have not been encoded yet.
# - Four books, i.e. four titles, one in the front (attribute n=1) and three in the body (n=1, 2, 3).


my $rawtext = shift @ARGV;


# chinese into arabic numbers

my $digit = "[一二三四五六七八九]";
my %arabic = ( 一 => 1, 二 => 2, 三 => 3, 四 => 4, 五 => 5, 六 => 6, 七 => 7, 八 => 8, 九 => 9 );  

sub chinese2arabic {
	my $ch = shift;
	if ($ch =~ m!($digit)十($digit)!) { return 10*$arabic{$1}+$arabic{$2}; } 
	if ($ch =~ m!($digit)十!) { return 10*$arabic{$1}; } 
	if ($ch =~ m!十($digit)!) { return 10+$arabic{$1}; } 
	if ($ch =~ m!十!) { return 10; }
	if ($ch =~ m!($digit)!) { return $arabic{$1}; }
	return "???";
}

my $moderndigit = "[一二三四五六七八九〇]";
my %modernarabic = ( 一 => 1, 二 => 2, 三 => 3, 四 => 4, 五 => 5, 六 => 6, 七 => 7, 八 => 8, 九 => 9, 〇 => 0 );  

sub modernchinese2arabic {
	my $ch = shift;
	my $result = 0;
	while ($ch ne "") { $result = 10 * $result + $modernarabic{substr($ch, 0, 1, "")}; }
	return $result;
}


# text-specific preparations 

my $number = "$digit|$digit?十$digit?";  # numbers from 1 to 99 (there is no higher number in any heading)
my $nth = "第";

my $headingType = "[界|求|論|題]"; 
my %englishHeadingType = ( 界 => "definition", 求 => "construction", 論 => "axiom", 題 => "problem" );


# global variables

my @text;

# counters
my $line;
my $pb = 0;
my %div = ( 界 => 0, 求 => 0, 論 => 0, 題 => 0); 
my $note = 0;

# booleans
my $inParagraph = 0;
my $lineStartsWithS = 0;
my $inDiv = 0;
my $inFig = 0;
my $inTb = 0;


# read in the raw text

open (RAWTEXT, "<:utf8", $rawtext) or die "Can't open the raw text file \"$rawtext\"!\n";
while (<RAWTEXT>) {

	s!\r\n!!;  # remove Windows line ends
	push @text, $_;
}
close (RAWTEXT);


# emendations of the raw text
# (see also the reduplicated ECHO pages in the processing of page breaks)

$line = 0;

for (@text) {

	$line++;

	# normalize the zero in modern page numbers
	s!○!〇!g;  # white circle U+25CB --> ideographic number zero U+3007
	
	# ignore outdentation in the preface
	if ($line < 153) { s!<p x>!<p>!; } 

	# pre-process notes that continue on the next line	
	if ($text[$line-1] =~ m!</sm>$!) {
		my $counterpart = $line;
		if ($text[$counterpart] =~ m!^<pb!) { $counterpart++; }
		if ($text[$counterpart] =~ m!^<sm>!) {	
			$text[$line-1] =~ s!</sm>$!!;
			$text[$counterpart] =~ s!^<sm>!!;
		}
	}
	
	# fill in the unknown characters (MSi)
	s!<001>!轂!g;  # s!<001>!<unknown code="001" unicode="8F42">轂</unknown>!g;
	s!<002>!<unknown code="002" unicode="2F88D">庶</unknown>!g; # the actual Unicode character 庶 breaks oXygen

	# clarify <?> (the list is not complete!)
	s!<？>!<?>!; # line 811: fullwidth question mark U+FF1F --> ASCII question mark U+003F
	s!餘<\?>二倣此。!餘二倣此。!; # line 811
	s!愈<\?>!愈!g; # MSi: the reading is correct
	s!。<\?>!。!;  # lines 1041, 1087, 2104, 3855: all unclear periods are plausible
	# (e.g. line 1041: MSi: It is in the middle of a sentence, but a period at this position is quite common nonetheless.) 
	s!有兩種幾何<\?>。!有兩種幾何。!; # line 2208
	s!其元<\?>大者!其元大者!; # line 2208 again

	# replace the @
	s!\@增至於無窮。!遞增至於無窮。!; # line 2208 again: U+905E
	s!\@減至於無窮。!遞減至於無窮。!; # line 2208 again: U+905E
	s!<p>@!<p><gap/>!; # lines 3642, 3667, 3815: missing lines of text (last lines on pages 278, 280 and 288)

	# missing line breaks (the list is not complete!)
	s!小於兩直角。則此二橫直線。!小於兩直角。則此二橫<lb/>直線。!; # line 403; may have to do with the neighboring figure
	s!俱小於直角。或幷之小於兩直角。!俱小於直角。或幷之小<lb/>於兩直角。!; # line 404
	
	# normalize the hash in the table
	s!＃!#!g; # fullwidth number sign U+FF03 --> ASCII hash, i.e. number sign U+0023
	
	# move the only table in the text (ECHO p.327) out of its surrounding sentence
	s!却云十六與十二之比例。若!却云十六與十二之比例。!;  # line 4562
	s!八與三、及二與四之比例。!若<lb/>八與三、及二與四之比例。!; #line 4573

	# misc. emendations
	s!N12<114608657010!N12x114608657010!; # line 5: replace "<" in library stamp junk
	s!<pb 六><h>幾何原本　卷一之首</h>!<pb 六><rh>幾何原本　卷一之首</rh>!;  # line 245 (obvious mistake)
	s!<h>後支前己正論</h>!<p>後支前己正論</p>!;  # line 2175 (Tian Miao: wrong tag)
	if ($line == 2992) { s!<h>第三十四題</h>!<h>第十四題</h>!; }  # line 2992 (obvious mistake)

}


# process the raw text

# The metadata add 31 lines to the text. Apart from these prefix lines, the line structure of the original is preserved.

    print <<'METADATA';
<?xml version="1.0" encoding="UTF-8"?>
<echo xmlns="http://www.mpiwg-berlin.mpg.de/ns/echo/1.0/"
      xmlns:dc="http://purl.org/dc/elements/1.1/"
      xmlns:dcq="http://purl.org/dc/qualifiers/1.0/"
      xmlns:dct="http://purl.org/dc/terms/1.0/"
      xmlns:echo="http://www.mpiwg-berlin.mpg.de/ns/echo/1.0/"
      xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
      xmlns:xhtml="http://www.w3.org/1999/xhtml"
      xmlns:xlink="http://www.w3.org/1999/xlink"
      xmlns:xml="http://www.w3.org/XML/1998/namespace">
  <metadata>
    <dc:creator>Euclid</dc:creator>
    <dc:title xml:lang="zh">Jihe yuanben</dc:title>
    <dc:title xml:lang="zh">幾何原本</dc:title>
    <dc:date>
      <rdf:Description>
        <dcq:dateScheme>ISO 8601</dcq:dateScheme>
        <rdf:value>1966</rdf:value>
      </rdf:Description>
    </dc:date>
    <dc:language>
      <rdf:Description>
        <dcq:languageScheme>ISO 639-2</dcq:languageScheme>
        <rdf:value>zho-Hant</rdf:value>
      </rdf:Description>
    </dc:language>
    <dc:identifier>test.xml</dc:identifier>
    <dc:rights>open access</dc:rights>
    <dct:license>http://echo.mpiwg-berlin.mpg.de/policy/oa_basics/declaration</dct:license>
  </metadata>
   <text xml:lang="zh">
METADATA


$line = 0;

for (@text) {

	$line++;
	
	# unknown characters
	# (must be before figures so that of <?> in a <var> will be processed)
	s!<\?>!<unsure/>!g;

	
	# figures
	# (must be before the tagging of div's to avoid </div></fig>)
	# (the internal ordering is important for successive figures)
	
	if ($inFig) {
		if (m!<cap>! || m!<desc>! || m!<var>!) { 
			print "$_\n";
			next;
		}
		print "</figure>";
		$inFig = 0;
	}
	if (s!<fig>!<figure>!) {   #	!<figure place="text">! 
		$inFig = 1; 
		print "$_\n";
		next;
	}


	# the table on ECHO p.327
	# (must be before paragraphs)
	
	if (m!<tb>!) { $inTb = 1; }

	if ($inTb) {
		unless (s!<(/?)tb>!<$1xhtml:table>!) {
			$_ = "<xhtml:tr><xhtml:td>$_</xhtml:td></xhtml:tr>";
			s!#!</xhtml:td><xhtml:td>!g;
		}
		print "$_\n";
		if (m!</xhtml:table>!) { $inTb = 0; }
		next;
	}


	# ad hoc tagging of book covers, preface, chapters, chapter heads, chapter mains, backmatter
	# including some additional lowest-level closing div's
	
	sub beginDiv {
		$inDiv = 0;
		foreach my $key (keys %div) { $div{$key} = 0; } 
	}
	
	if ( $line == 1) { print '<front><div level="1" type="title" n="1">'; }  # 1 - 32
	if ( $line == 33) { print '</div>'; }

	if ( $line == 33) { print '<div level="1" type="preface" n="1">'; } # 33 - 152
	if ( $line == 153) { print '</div></front>'; }

	if ( $line == 153) { print '<body><div level="1" type="chapter" n="1"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
	if ( $line == 452) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
	if ( $line == 1364) { print '</div></div></div>'; }

	if ( $line == 1364) {print '<div type="title" n="1">'; } # 1364 - 1385
	if ( $line == 1386) { print '</div>'; }

	if ( $line == 1386) { print '<div level="1" type="chapter" n="2"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
	if ( $line == 1416) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
	if ( $line == 1753) { print '</div></div></div>'; }

	if ( $line == 1753) { print '<div level="1" type="chapter" n="3"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
	if ( $line == 1848) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
	if ( $line == 2722) { print '</div></div></div>'; }

	if ( $line == 2722) { print '<div type="title" n="2">'; } # 2722 - 2744
	if ( $line == 2745) { print '</div>'; }

	if ( $line == 2745) { print '<div level="1" type="chapter" n="4"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
	if ( $line == 2787) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
	if ( $line == 3109) { print '</div></div></div>'; }

	if ( $line == 3109) { print '<div level="1" type="chapter" n="5"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
	if ( $line == 3520) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
	if ( $line == 4387) { print '</div></div></div>'; }

	if ( $line == 4387) { print '<div type="title" n="3">'; } # 4387 - 4392
	if ( $line == 4393) { print '</div>'; }

	if ( $line == 4393) { print '<div level="1" type="chapter" n="6"><div level="2" type="chapterhead" n="1">'; beginDiv; } 
	if ( $line == 4594) { print '</div></div><div level="2" type="chaptermain" n="1">'; beginDiv; } 
	if ( $line == 5984) { print '</div></div></div></body>'; }

	if ( $line == 5984) { print '<back><div type="backmatter">'; } # 5984 - end; </div></back> is printed together with </xml>, see below.


	# page breaks

	if (m!<pb!) {
		$pb++;
		
#		# The ECHO pages 215 to 220 reduplicate the pages 209 to 214 and have been typed only once by Formax
#		if ($pb == 215) { $pb = 221; } 
		
		m!^<pb( ($moderndigit+))?>(<rh>(.+)</rh>)? *$!;
		my $pagenumber = ($2 ? " o=\"$2\" oE=\"" . &modernchinese2arabic($2). "\"" : "");
		my $runninghead = ($4 ? " rhead=\"$4\"" : "" );
		my $link = 'http://echo.mpiwg-berlin.mpg.de/ECHOdocuView/ECHOzogiLib?'.
			'url=/mpiwg/online/permanent/library/02NT95YF/pageimg&amp;mode=imagepath&amp;pn='.$pb;
		$_ = "<pb$pagenumber$runninghead n=\"$pb\" xlink:href='$link'/>";
		print "$_\n";
		next;
	}

	
	# headings

	if (m!<h>!) { 

		s!<h>!<head>!;
		s!</h>!</head>!;

		# notes in headings
		s!　?\\\\　?!<lb type="halfline"/>!g;
		s!<sm>(.+)</sm>!<note>$1</note>!;

		# headings at the lowest level
		if (m!<head>$nth($number)($headingType).*?</head>$!)  { 
			my $o = $1;
			my $oE = &chinese2arabic($o);
			my $type = $2;
			my $typeE = $englishHeadingType{$2};
			s!　(.+?)</head>!<note>$1</note></head>!;		
			$div{$type}++;
			if ($div{$type} != $oE) { die "$line: The numbering of the div's is incorrect!\n"; }
			
			if ($inDiv) { print "</div>"; }
			print '<div type="'.$type.'" typeE="'.$typeE.'" o="'.$o.'" level="3" n="'.$div{$type}.'">'; 
			$inDiv = 1;
		} 
		
		# all headings
		print "$_\n";
		next;
	}


	# paragraphs
	# (<p> is always at the beginning, </p> always at the end of a line)
	# inside a paragraph: <pb ...>, <rh>; <sm>; <fig>, <desc>, <var>; 1x <tb>; 3x <002> 
	
	my $outdented = s!<p x>!<p>!;

	if (m!<p>!) { $inParagraph = 1; }
	
	if ($inParagraph) {

		# normalize the periods: there should always be a period before </p> and </sm>
		# (if the period is missing, insert an ASCII period)
		s!([^。])</p>!$1\.</p>!g;
		s!</sm>\.</p>!</sm></p>!;
		s!([^。])<sm>!$1\.<sm>!g;
		s!^\.<sm>!<sm>!;
		s!<p>\.<sm>!<p><sm>!;
		s!([^。])</sm>!$1\.</sm>!g;
	
		# turn small text into notes
		s!<sm>!<note>!g;  # <sm> and </sm> need not be in the same line
		s!</sm>!</note>!g;
		s!　?\\\\　?!<lb type="halfline"/>!g;

		# tag sentences
		s!<p>!<p><s>!;
		if ($lineStartsWithS) { $_ = "<s>$_"; $lineStartsWithS = 0; } 
		s![。.]!$&</s><s>!g;
		if (s!<s>$!!) { $lineStartsWithS = 1; }
		s!<s><note>!<note><s>!g;      #  ???
		s!<s></note>!</note><s>!g;
		s!<s><lb type="halfline"/>!<lb type="halfline"/><s>!g;
		s!<s></p>!</p>!;
		
		# end each line with <lb/>
		$_ .= "<lb/>" unless (($line < $#text) &&(($text[$line] =~ m!<tb>!) || ($text[$line] =~ m!<pb!)));
		s!</p><lb/>!</p>!;
		s!<s><lb/>!<lb/><s>!g;
		
		# outdented paragraphs
		if ($outdented) { s!<p>!<p outdented="1">!; }
		
		print "$_\n";

		if (m!</p>!) { $inParagraph = 0; }
		next;
	}
	

	# everything else
	
	print "$_\n";	
}


print "</div>";  # end of backmatter
print "</back></text></echo>\n";


