--- FM2SQL/Attic/Convert.java 2004/08/02 11:39:23 1.65
+++ FM2SQL/Attic/Convert.java 2004/08/08 12:01:41 1.70
@@ -29,10 +29,11 @@ import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
+import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
-import java.util.Properties;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.Vector;
@@ -40,8 +41,34 @@ import java.util.Vector;
import com.exploringxml.xml.Node;
import com.exploringxml.xml.Xparse;
+
class Convert
{
+ /**Helper class for index creation
+ * @author rogo
+ *
+ */
+ public static class IndexList extends Vector
+ {
+ public String toString()
+ {
+ StringBuffer buff = new StringBuffer(1000);
+ int count = 0;
+ for (Iterator iter = this.iterator(); iter.hasNext();)
+ {
+ String element = (String) iter.next();
+ if (count < elementCount-1)
+ {
+ buff.append(element).append(", ");
+ count++;
+ } else
+ buff.append(element);
+
+ }
+
+ return buff.toString();
+ }
+ }
static DBBean bean = new DBBean();
static DBBean beanDest = new DBBean();
@@ -1126,6 +1153,7 @@ class Convert
Vector selects = new Vector();
Vector creates = new Vector();
Vector ids = new Vector();
+ Vector indexListVec= new Vector();
String delimiter = "|";
int mode = -1;
@@ -1145,6 +1173,7 @@ class Convert
selects = new Vector();
creates = new Vector();
ids = new Vector();
+ indexListVec= new Vector();
// parse dataBase
Node node = root.find("convert/source/database/url", new int[] { 1, 1, i, 1 });
Node node1 = root.find("convert/source/database/user", new int[] { 1, 1, i, 1, 1 });
@@ -1219,6 +1248,8 @@ class Convert
String name = (String) node4.attributes.get("name");
String layout = (String) node4.attributes.get("layout");
String id = (String) node4.attributes.get("id");
+ String indexList = (String) node4.attributes.get("indexList");
+
System.out.println("id was " + id);
if (name == null)
throw new Error("parse error required table tag attribute name missing");
@@ -1226,11 +1257,15 @@ class Convert
layout = "";
if (id == null)
id = "";
+ if(indexList==null)
+ indexList="";
if (name.equals(""))
throw new Error("parse error table tag attribute must not be empty");
tables.add(name);
layouts.add(layout);
+
ids.add(id);
+ indexListVec.add(indexList);
String query = (node5 == null) ? "" : node5.getCharacters();
if (query.equals(""))
System.err.println("Warning empty select tag or select tag missing !!");
@@ -1244,6 +1279,7 @@ class Convert
}
DataBase dataBase = new DataBase(database, tables, layouts, selects, creates, ids, mode);
dataBase.delimiter = delimiter;
+ dataBase.buildIndexTable(indexListVec);
databases.add(dataBase);
}
DBBean database = new DBBean();
@@ -1272,7 +1308,7 @@ class Convert
userDest = database.user;
passwdDest = database.passwd;
- synchronize(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode);
+ synchronize(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode, db.delimiter);
}
}
// printContents(node3);
@@ -1574,6 +1610,10 @@ class Convert
Vector tables;
Vector ids;
String delimiter = "//";
+ /**
+ * maps table name to index fields
+ */
+ Hashtable htIndex = new Hashtable();
boolean useNormanToUnicodeMapper = false;
final static int CONVERT_MODE = 1;
@@ -1596,6 +1636,16 @@ class Convert
this.bean.setIDVector(ids);
}
/**
+ * @param indexListVec
+ */
+ public void buildIndexTable(Vector indexListVec)
+ {
+ for (int i = 0; i < tables.size(); i++)
+ {
+ fillIndexList((String)tables.get(i),(String)indexListVec.get(i));
+ }
+ }
+ /**
* writes the data contained in this object to the buffered writer
* * @param buffr
* @throws Exception
@@ -1629,8 +1679,10 @@ class Convert
String select = (String) selects.get(index);
String create = (String) creates.get(index);
String id = (String) ids.get(index);
-
- buffr.write("
\n");
+ IndexList indexList = (IndexList) htIndex.get(table);
+ if (indexList == null)
+ indexList = new IndexList();
+ buffr.write(" \n");
buffr.write(" \n");
if (!create.equals(""))
buffr.write(" " + create + " \n");
@@ -1639,6 +1691,19 @@ class Convert
}
buffr.write(" \n");
}
+
+ public void fillIndexList(String table, String list)
+ {
+ IndexList indexList = new IndexList();
+ StringTokenizer tokenizer = new StringTokenizer(list, ",");
+ while (tokenizer.hasMoreTokens())
+ {
+ indexList.add(tokenizer.nextToken());
+ }
+ System.out.println(indexList);
+
+ htIndex.put(table, indexList);
+ }
public String toString()
{
return bean.url + " " + tables;
@@ -1896,7 +1961,7 @@ class Convert
*/
// TODO implement append,update and delete in one method
// TODO using id based algorithm
- public static void synchronize(String source, String destination, Vector names, Vector layouts, Vector selects, Vector creates, Vector ids, int mode) throws Exception
+ public static void synchronize(String source, String destination, Vector names, Vector layouts, Vector selects, Vector creates, Vector ids, int mode, String delimiter) throws Exception
{
System.out.println(" bin in synchronize!!!");
FM2SQL.ProgressDialog dialog = null;
@@ -2000,6 +2065,8 @@ class Convert
{
long startTime = System.currentTimeMillis();
int counter = -1;
+ TreeSet linesToDelete = null;
+ PreparedStatement delPSt = null;
while (true)
{
++counter;
@@ -2007,8 +2074,9 @@ class Convert
dialog.title.setText("Check if data is available");
else if (dialog != null)
dialog.title.setText("Check if more data is available");
- myIds = bean.getIDVector(ids.get(tbIndex).toString(), (String) names.get(tbIndex), tempQuery, numHits);
- myIdsDest = beanDest.getIDVector(convertText(idField), destTableName, tempQueryDest, numHits);
+ myIds = bean.getIDVector(ids.get(tbIndex).toString(), (String) names.get(tbIndex), tempQuery, 0);
+ myIdsDest = beanDest.getIDVector(convertText(idField), destTableName, tempQueryDest, 0);
+ //System.out.println("status of remove "+myIds.remove("b015892"));
if (myIds.isEmpty())
break;
vec = new Vector(myIds);
@@ -2016,141 +2084,99 @@ class Convert
rowCount = vec.size();
// Deletion will work this way
Vector deleted = new Vector(vec);
- TreeSet linesToDelete = new TreeSet(vecDest);
TreeSet linesToAppend = new TreeSet(vec);
+ linesToAppend.addAll(vec);
+ linesToDelete = new TreeSet(vecDest);
// remove all lines that are already in dest database
linesToAppend.removeAll(vecDest);
// remove all lines that should not be deleted
linesToDelete.removeAll(deleted);
- System.out.println("linesToAppend " + linesToAppend + " " + vecDest + " " + destTableName);
- System.out.println("linesToDelete " + linesToDelete + " " + vecDest + " " + destTableName);
- // prepare new query for next chunk
- if (query.indexOf("where") > 0)
- tempQuery = query + " and " + tempID + ">='" + vec.firstElement() + "' and " + tempID + "<='" + vec.lastElement() + "' order by " + tempID;
- else
- tempQuery = query + " where " + tempID + ">='" + vec.firstElement() + "' and " + tempID + "<='" + vec.lastElement() + "'"; // order by "+tempID;
- System.out.println("tempQuery is now " + tempQuery + " order by " + tempID);
- // bean.makeQuery(tempQuery,vec.size());
-
+ System.out.println("linesToAppend " + linesToAppend + " " + destTableName);
+ System.out.println("linesToDelete " + linesToDelete + " " + destTableName);
+ System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIds.size());
+ deltaID = (int) Math.round((double) myIds.size() / (double) numIntervalls);
ConversionProperties prop = getFieldNamesAndDestTableName(creates.get(tbIndex).toString(), query, names.get(tbIndex).toString());
StringBuffer insCommand = createInsertCommand(prop.destTableName, prop.fieldNames);
- StringBuffer updCommand = createUpdateCommand(prop.destTableName, prop.fieldNames, tempID);
- StringBuffer delCommand = createDeleteCommand(destTableName,tempID);
+ StringBuffer updCommand = createUpdateCommand(prop.destTableName, prop.fieldNames, tempIDdest);
+ StringBuffer delCommand = createDeleteCommand(destTableName, tempIDdest);
PreparedStatement insPst = beanDest.getConnection().prepareStatement(insCommand.toString());
PreparedStatement updPst = beanDest.getConnection().prepareStatement(updCommand.toString());
- PreparedStatement delPSt = beanDest.getConnection().prepareStatement(delCommand.toString());
- Vector[] vectors = bean.getQueryData(tempQuery, vec.size());
- int idIndex = vectors[1].indexOf(idField);
- // todo arraylist code has to be added
- for (Iterator iter = vectors[0].iterator(); iter.hasNext();)
+ delPSt = beanDest.getConnection().prepareStatement(delCommand.toString());
+ // delPSt.setString(1,"b015892");
+ // delPSt.execute();
+ // if (true)
+ // return;
+ if (vec.size() <= numIntervalls)
{
- Vector line = (Vector) iter.next();
- Object lineIDIndex = line.get(idIndex);
- System.out.println("line " + linesToDelete.contains(line.get(idIndex))+" "+lineIDIndex);
- if (linesToAppend.contains(lineIDIndex))
- {
- for (int l = 0; l < line.size(); ++l)
- {
- Object obj = line.get(l);
- if (obj != null)
- insPst.setString(l + 1, obj.toString());
- else
- insPst.setNull(l + 1, Types.NULL);
- }
- insPst.execute();
-
- }
-
- else // update
- {
- for (int l = 0; l < line.size(); ++l)
- {
- Object obj = line.get(l);
- if (obj != null)
- updPst.setString(l + 1, obj.toString());
- else
- updPst.setNull(l + 1, Types.NULL);
- }
- updPst.setString(line.size()+1,line.get(idIndex).toString());
- updPst.execute();
-
- }
+ endIndex = 0;
+ deltaID = vec.size();
}
-
- // delPSt.setString(1,"6");
- // delPSt.setString(1,"7");
- // delPSt.setString(1,"8");
-
- //delPSt.execute();
- System.out.println();
- // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size());
- /// @TODO complete delete task remove query show lines to be deleted let user choose if he wants that
- System.out.println("number of lines to be deleted " + linesToDelete.size());
- deltaID = (int) Math.round((double) myIds.size() / (double) numIntervalls);
- System.out.println("deltaID " + deltaID);
- //System.out.println();
-
- if (true)
- return;
-
- beanDest.setConnection(destination);
-
- Statement stm = beanDest.getConnection().createStatement();
-
- Vector tables = beanDest.getTableNames();
- // Collections.sort(tables);
- System.out.println(names.get(tbIndex) + " " + tables.indexOf(convertText((String) names.get(tbIndex)))); // "//beanDest.getTypeNames());
- tables = beanDest.getTableNames();
- // System.out.println(beanDest.getTableNames(beanDest.getCatalogs().get(2).toString()));
- stm = beanDest.getConnection().createStatement();
-
- if (dialog != null)
- dialog.title.setText(" Deleting table data ...");
-
- int j = -1;
+ for (int k = 0; k < vec.size() - deltaID; k = k + deltaID)
+ {
+ System.out.println(vec.get(k) + " " + vec.get(k + deltaID) + " " + vec.lastElement());
+ if (query.indexOf("where") > 0)
+ tempQuery = query + " and " + tempID + ">='" + vec.get(k) + "' and " + tempID + "<='" + vec.get(k + deltaID) + "'";
+ else
+ tempQuery = query + " where " + tempID + ">='" + vec.get(k) + "' and " + tempID + "<='" + vec.get(k + deltaID) + "'";
+ System.out.println(tempQuery);
+ if (dialog != null)
+ dialog.title.setText("Reading table data ...");
- Vector row = null;
- command = new StringBuffer();
+ // bean.makeQuery(tempQuery, deltaID);
+ if (dialog != null)
+ dialog.title.setText("Writing table data ...");
- command.append("DELETE FROM");
- command.append(beanDest.getQC());
- command.append(destTableName);
- //command.append(convertText((String) names.get(tbIndex)));
- command.append(beanDest.getQC());
- int size = bean.getColumnNames().size();
- command.append("WHERE " + convertText(ids.get(tbIndex).toString()) + " = ?");
- PreparedStatement pstm = beanDest.getConnection().prepareStatement(command.toString());
- System.out.println(command + " " + tbIndex);
- //int rowCount = bean.getRowCount(query);
- // int idIndex = bean.getColumnNames().indexOf(ids.get(tbIndex));
-/* while (true)
+ performSynchronize(idField, vec, tempQuery, linesToDelete, linesToAppend, insPst, updPst, delPSt, deltaID, delimiter);
+ // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size());
+ endIndex = k + deltaID;
+ }
+ System.out.println(endIndex);
+ //all data written ? if not write last chunk of data
+ if (endIndex == vec.size() - 1)
+ System.out.println("fits");
+ else
{
+ System.out.println(" last intervall from " + vec.get(endIndex) + " " + vec.lastElement());
- ++j;
- if (j == linesToDelete.size())
- break;
- //print rows
- pstm.setString(1, linesToDelete.get(j).toString());
- System.out.println(pstm.toString());
- pstm.execute();
+ if (query.indexOf("where") > 0)
+ tempQuery = query + " and " + tempID + ">='" + vec.get(endIndex) + "' and " + tempID + "<='" + vec.lastElement() + "'";
+ else
+ tempQuery = query + " where " + tempID + ">='" + vec.get(endIndex) + "' and " + tempID + "<='" + vec.lastElement() + "'";
+ System.out.println(tempQuery);
if (dialog != null)
- dialog.progress.setValue((int) (((double) (j + 1) / (double) rowCount) * 100.0));
- command = null;
+ dialog.title.setText("Reading table data ...");
+ // bean.makeQuery(tempQuery, 0);
+ if (dialog != null)
+ dialog.title.setText("Writing table data ...");
+ performSynchronize(idField, vec, tempQuery, linesToDelete, linesToAppend, insPst, updPst, delPSt, deltaID, delimiter);
+ // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size());
}
// prepare new query for next chunk
if (query.indexOf("where") > 0)
tempQuery = query + " and " + tempID + ">'" + vec.lastElement() + "'";
else
tempQuery = query + " where " + tempID + ">'" + vec.lastElement() + "'";
-*/
- } //to outer while
+
+ }
+
+ for (Iterator iter = linesToDelete.iterator(); iter.hasNext();)
+ {
+ String id = (String) iter.next();
+ delPSt.setString(1, id);
+ delPSt.execute();
+
+ }
+
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time for incremental synchronize elapsed " + (endTime - startTime));
} // to idfield if
} // table loop
} catch (Exception e)
{
System.out.println("Error while connecting to database " + e);
+ e.printStackTrace();
if (dialog != null)
{
dialog.setVisible(false);
@@ -2171,6 +2197,61 @@ class Convert
dialog.setVisible(false);
}
+ }
+ private static void performSynchronize(
+ String idField,
+ Vector vec,
+ String tempQuery,
+ TreeSet linesToDelete,
+ TreeSet linesToAppend,
+ PreparedStatement insPst,
+ PreparedStatement updPst,
+ PreparedStatement delPSt,
+ int deltaID,
+ String delimiter)
+ throws SQLException, ParseException
+ {
+ Vector[] vectors = bean.getQueryData(tempQuery, deltaID);
+ int idIndex = vectors[1].indexOf(idField);
+ // todo arraylist code has to be added
+ for (Iterator iter = vectors[0].iterator(); iter.hasNext();)
+ {
+ Vector line = (Vector) iter.next();
+ Object lineIDIndex = line.get(idIndex);
+ if (linesToAppend.contains(lineIDIndex))
+ System.out.println("line " + linesToAppend.contains(line.get(idIndex)) + " " + lineIDIndex);
+ if (linesToAppend.contains(lineIDIndex))
+ {
+ for (int l = 0; l < line.size(); ++l)
+ {
+ Object obj = line.get(l);
+ if (obj instanceof ArrayList)
+ obj = formatFileMakerArray((List) obj, delimiter);
+ if (obj != null)
+ insPst.setString(l + 1, obj.toString());
+ else
+ insPst.setNull(l + 1, Types.NULL);
+ }
+ insPst.execute();
+
+ } else // update
+ {
+ for (int l = 0; l < line.size(); ++l)
+ {
+ Object obj = line.get(l);
+ if (obj instanceof ArrayList)
+ obj = formatFileMakerArray((List) obj, delimiter);
+ if (obj != null)
+ updPst.setString(l + 1, obj.toString());
+ else
+ updPst.setNull(l + 1, Types.NULL);
+ }
+ updPst.setString(line.size() + 1, line.get(idIndex).toString());
+ updPst.execute();
+
+ }
+ }
+
} // to method
/**
@@ -3692,7 +3773,7 @@ class Convert
while (tokenizer.hasMoreTokens())
{
String fieldName = tokenizer.nextToken().trim();
- fieldNames[fieldIndex] = convertText(fieldName);
+ fieldNames[fieldIndex] =beanDest.getQC() + convertText(fieldName)+beanDest.getQC();
// System.out.println("field "+ fieldNames[fieldIndex]);
fieldIndex++;
}
@@ -3705,7 +3786,7 @@ class Convert
for (Iterator iter = fieldNamesVec.iterator(); iter.hasNext();)
{
String element = (String) iter.next();
- fieldNames[++fieldIndex] = bean.getQC() + convertText(element) + bean.getQC();
+ fieldNames[++fieldIndex] = beanDest.getQC() + convertText(element) + beanDest.getQC();
// System.out.println("field " + fieldNames[fieldIndex]);
}
}
@@ -3770,6 +3851,6 @@ class Convert
//command.append(convertText((String) names.get(tbIndex)));
command.append(beanDest.getQC());
command.append("WHERE " + idField + " = ?");
- return command;
+ return command;
}
}
\ No newline at end of file