--- FM2SQL/Attic/Convert.java 2004/08/02 11:39:23 1.65 +++ FM2SQL/Attic/Convert.java 2004/08/06 11:15:41 1.69 @@ -29,10 +29,11 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; +import java.text.ParseException; import java.util.ArrayList; +import java.util.Hashtable; import java.util.Iterator; import java.util.List; -import java.util.Properties; import java.util.StringTokenizer; import java.util.TreeSet; import java.util.Vector; @@ -40,8 +41,34 @@ import java.util.Vector; import com.exploringxml.xml.Node; import com.exploringxml.xml.Xparse; + class Convert { + /**Helper class for index creation + * @author rogo + * + */ + public static class IndexList extends Vector + { + public String toString() + { + StringBuffer buff = new StringBuffer(1000); + int count = 0; + for (Iterator iter = this.iterator(); iter.hasNext();) + { + String element = (String) iter.next(); + if (count < elementCount-1) + { + buff.append(element).append(", "); + count++; + } else + buff.append(element); + + } + + return buff.toString(); + } + } static DBBean bean = new DBBean(); static DBBean beanDest = new DBBean(); @@ -1126,6 +1153,7 @@ class Convert Vector selects = new Vector(); Vector creates = new Vector(); Vector ids = new Vector(); + Vector indexListVec= new Vector(); String delimiter = "|"; int mode = -1; @@ -1145,6 +1173,7 @@ class Convert selects = new Vector(); creates = new Vector(); ids = new Vector(); + indexListVec= new Vector(); // parse dataBase Node node = root.find("convert/source/database/url", new int[] { 1, 1, i, 1 }); Node node1 = root.find("convert/source/database/user", new int[] { 1, 1, i, 1, 1 }); @@ -1219,6 +1248,8 @@ class Convert String name = (String) node4.attributes.get("name"); String layout = (String) node4.attributes.get("layout"); String id = (String) node4.attributes.get("id"); + String indexList = (String) node4.attributes.get("indexList"); + System.out.println("id was " + id); if (name == null) throw new Error("parse error required table tag attribute name missing"); @@ -1226,11 +1257,15 @@ class Convert layout = ""; if (id == null) id = ""; + if(indexList==null) + indexList=""; if (name.equals("")) throw new Error("parse error table tag attribute must not be empty"); tables.add(name); layouts.add(layout); + ids.add(id); + indexListVec.add(indexList); String query = (node5 == null) ? "" : node5.getCharacters(); if (query.equals("")) System.err.println("Warning empty select tag or select tag missing !!"); @@ -1244,6 +1279,7 @@ class Convert } DataBase dataBase = new DataBase(database, tables, layouts, selects, creates, ids, mode); dataBase.delimiter = delimiter; + dataBase.buildIndexTable(indexListVec); databases.add(dataBase); } DBBean database = new DBBean(); @@ -1272,7 +1308,7 @@ class Convert userDest = database.user; passwdDest = database.passwd; - synchronize(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode); + synchronize(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode, db.delimiter); } } // printContents(node3); @@ -1574,6 +1610,10 @@ class Convert Vector tables; Vector ids; String delimiter = "//"; + /** + * maps table name to index fields + */ + Hashtable htIndex = new Hashtable(); boolean useNormanToUnicodeMapper = false; final static int CONVERT_MODE = 1; @@ -1596,6 +1636,16 @@ class Convert this.bean.setIDVector(ids); } /** + * @param indexListVec + */ + public void buildIndexTable(Vector indexListVec) + { + for (int i = 0; i < tables.size(); i++) + { + fillIndexList((String)tables.get(i),(String)indexListVec.get(i)); + } + } + /** * writes the data contained in this object to the buffered writer * * @param buffr * @throws Exception @@ -1629,8 +1679,10 @@ class Convert String select = (String) selects.get(index); String create = (String) creates.get(index); String id = (String) ids.get(index); - - buffr.write(" \n"); + IndexList indexList = (IndexList) htIndex.get(table); + if (indexList == null) + indexList = new IndexList(); + buffr.write("
\n"); buffr.write(" \n"); if (!create.equals("")) buffr.write(" " + create + " \n"); @@ -1639,6 +1691,19 @@ class Convert } buffr.write(" \n"); } + + public void fillIndexList(String table, String list) + { + IndexList indexList = new IndexList(); + StringTokenizer tokenizer = new StringTokenizer(list, ","); + while (tokenizer.hasMoreTokens()) + { + indexList.add(tokenizer.nextToken()); + } + System.out.println(indexList); + + htIndex.put(table, indexList); + } public String toString() { return bean.url + " " + tables; @@ -1896,7 +1961,7 @@ class Convert */ // TODO implement append,update and delete in one method // TODO using id based algorithm - public static void synchronize(String source, String destination, Vector names, Vector layouts, Vector selects, Vector creates, Vector ids, int mode) throws Exception + public static void synchronize(String source, String destination, Vector names, Vector layouts, Vector selects, Vector creates, Vector ids, int mode, String delimiter) throws Exception { System.out.println(" bin in synchronize!!!"); FM2SQL.ProgressDialog dialog = null; @@ -2000,6 +2065,8 @@ class Convert { long startTime = System.currentTimeMillis(); int counter = -1; + TreeSet linesToDelete = null; + PreparedStatement delPSt = null; while (true) { ++counter; @@ -2007,8 +2074,9 @@ class Convert dialog.title.setText("Check if data is available"); else if (dialog != null) dialog.title.setText("Check if more data is available"); - myIds = bean.getIDVector(ids.get(tbIndex).toString(), (String) names.get(tbIndex), tempQuery, numHits); - myIdsDest = beanDest.getIDVector(convertText(idField), destTableName, tempQueryDest, numHits); + myIds = bean.getIDVector(ids.get(tbIndex).toString(), (String) names.get(tbIndex), tempQuery, 0); + myIdsDest = beanDest.getIDVector(convertText(idField), destTableName, tempQueryDest, 0); + //System.out.println("status of remove "+myIds.remove("b015892")); if (myIds.isEmpty()) break; vec = new Vector(myIds); @@ -2016,141 +2084,99 @@ class Convert rowCount = vec.size(); // Deletion will work this way Vector deleted = new Vector(vec); - TreeSet linesToDelete = new TreeSet(vecDest); TreeSet linesToAppend = new TreeSet(vec); + linesToAppend.addAll(vec); + linesToDelete = new TreeSet(vecDest); // remove all lines that are already in dest database linesToAppend.removeAll(vecDest); // remove all lines that should not be deleted linesToDelete.removeAll(deleted); - System.out.println("linesToAppend " + linesToAppend + " " + vecDest + " " + destTableName); - System.out.println("linesToDelete " + linesToDelete + " " + vecDest + " " + destTableName); - // prepare new query for next chunk - if (query.indexOf("where") > 0) - tempQuery = query + " and " + tempID + ">='" + vec.firstElement() + "' and " + tempID + "<='" + vec.lastElement() + "' order by " + tempID; - else - tempQuery = query + " where " + tempID + ">='" + vec.firstElement() + "' and " + tempID + "<='" + vec.lastElement() + "'"; // order by "+tempID; - System.out.println("tempQuery is now " + tempQuery + " order by " + tempID); - // bean.makeQuery(tempQuery,vec.size()); - + System.out.println("linesToAppend " + linesToAppend + " " + destTableName); + System.out.println("linesToDelete " + linesToDelete + " " + destTableName); + System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIds.size()); + deltaID = (int) Math.round((double) myIds.size() / (double) numIntervalls); ConversionProperties prop = getFieldNamesAndDestTableName(creates.get(tbIndex).toString(), query, names.get(tbIndex).toString()); StringBuffer insCommand = createInsertCommand(prop.destTableName, prop.fieldNames); - StringBuffer updCommand = createUpdateCommand(prop.destTableName, prop.fieldNames, tempID); - StringBuffer delCommand = createDeleteCommand(destTableName,tempID); + StringBuffer updCommand = createUpdateCommand(prop.destTableName, prop.fieldNames, tempIDdest); + StringBuffer delCommand = createDeleteCommand(destTableName, tempIDdest); PreparedStatement insPst = beanDest.getConnection().prepareStatement(insCommand.toString()); PreparedStatement updPst = beanDest.getConnection().prepareStatement(updCommand.toString()); - PreparedStatement delPSt = beanDest.getConnection().prepareStatement(delCommand.toString()); - Vector[] vectors = bean.getQueryData(tempQuery, vec.size()); - int idIndex = vectors[1].indexOf(idField); - // todo arraylist code has to be added - for (Iterator iter = vectors[0].iterator(); iter.hasNext();) + delPSt = beanDest.getConnection().prepareStatement(delCommand.toString()); + // delPSt.setString(1,"b015892"); + // delPSt.execute(); + // if (true) + // return; + if (vec.size() <= numIntervalls) { - Vector line = (Vector) iter.next(); - Object lineIDIndex = line.get(idIndex); - System.out.println("line " + linesToDelete.contains(line.get(idIndex))+" "+lineIDIndex); - if (linesToAppend.contains(lineIDIndex)) - { - for (int l = 0; l < line.size(); ++l) - { - Object obj = line.get(l); - if (obj != null) - insPst.setString(l + 1, obj.toString()); - else - insPst.setNull(l + 1, Types.NULL); - } - insPst.execute(); - - } - - else // update - { - for (int l = 0; l < line.size(); ++l) - { - Object obj = line.get(l); - if (obj != null) - updPst.setString(l + 1, obj.toString()); - else - updPst.setNull(l + 1, Types.NULL); - } - updPst.setString(line.size()+1,line.get(idIndex).toString()); - updPst.execute(); - - } + endIndex = 0; + deltaID = vec.size(); } - - // delPSt.setString(1,"6"); - // delPSt.setString(1,"7"); - // delPSt.setString(1,"8"); - - //delPSt.execute(); - System.out.println(); - // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size()); - /// @TODO complete delete task remove query show lines to be deleted let user choose if he wants that - System.out.println("number of lines to be deleted " + linesToDelete.size()); - deltaID = (int) Math.round((double) myIds.size() / (double) numIntervalls); - System.out.println("deltaID " + deltaID); - //System.out.println(); - - if (true) - return; - - beanDest.setConnection(destination); - - Statement stm = beanDest.getConnection().createStatement(); - - Vector tables = beanDest.getTableNames(); - // Collections.sort(tables); - System.out.println(names.get(tbIndex) + " " + tables.indexOf(convertText((String) names.get(tbIndex)))); // "//beanDest.getTypeNames()); - tables = beanDest.getTableNames(); - // System.out.println(beanDest.getTableNames(beanDest.getCatalogs().get(2).toString())); - stm = beanDest.getConnection().createStatement(); - - if (dialog != null) - dialog.title.setText(" Deleting table data ..."); - - int j = -1; + for (int k = 0; k < vec.size() - deltaID; k = k + deltaID) + { + System.out.println(vec.get(k) + " " + vec.get(k + deltaID) + " " + vec.lastElement()); + if (query.indexOf("where") > 0) + tempQuery = query + " and " + tempID + ">='" + vec.get(k) + "' and " + tempID + "<='" + vec.get(k + deltaID) + "'"; + else + tempQuery = query + " where " + tempID + ">='" + vec.get(k) + "' and " + tempID + "<='" + vec.get(k + deltaID) + "'"; + System.out.println(tempQuery); + if (dialog != null) + dialog.title.setText("Reading table data ..."); - Vector row = null; - command = new StringBuffer(); + // bean.makeQuery(tempQuery, deltaID); + if (dialog != null) + dialog.title.setText("Writing table data ..."); - command.append("DELETE FROM"); - command.append(beanDest.getQC()); - command.append(destTableName); - //command.append(convertText((String) names.get(tbIndex))); - command.append(beanDest.getQC()); - int size = bean.getColumnNames().size(); - command.append("WHERE " + convertText(ids.get(tbIndex).toString()) + " = ?"); - PreparedStatement pstm = beanDest.getConnection().prepareStatement(command.toString()); - System.out.println(command + " " + tbIndex); - //int rowCount = bean.getRowCount(query); - // int idIndex = bean.getColumnNames().indexOf(ids.get(tbIndex)); -/* while (true) + performSynchronize(idField, vec, tempQuery, linesToDelete, linesToAppend, insPst, updPst, delPSt, deltaID, delimiter); + // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size()); + endIndex = k + deltaID; + } + System.out.println(endIndex); + //all data written ? if not write last chunk of data + if (endIndex == vec.size() - 1) + System.out.println("fits"); + else { + System.out.println(" last intervall from " + vec.get(endIndex) + " " + vec.lastElement()); - ++j; - if (j == linesToDelete.size()) - break; - //print rows - pstm.setString(1, linesToDelete.get(j).toString()); - System.out.println(pstm.toString()); - pstm.execute(); + if (query.indexOf("where") > 0) + tempQuery = query + " and " + tempID + ">='" + vec.get(endIndex) + "' and " + tempID + "<='" + vec.lastElement() + "'"; + else + tempQuery = query + " where " + tempID + ">='" + vec.get(endIndex) + "' and " + tempID + "<='" + vec.lastElement() + "'"; + System.out.println(tempQuery); if (dialog != null) - dialog.progress.setValue((int) (((double) (j + 1) / (double) rowCount) * 100.0)); - command = null; + dialog.title.setText("Reading table data ..."); + // bean.makeQuery(tempQuery, 0); + if (dialog != null) + dialog.title.setText("Writing table data ..."); + performSynchronize(idField, vec, tempQuery, linesToDelete, linesToAppend, insPst, updPst, delPSt, deltaID, delimiter); + // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size()); } // prepare new query for next chunk if (query.indexOf("where") > 0) tempQuery = query + " and " + tempID + ">'" + vec.lastElement() + "'"; else tempQuery = query + " where " + tempID + ">'" + vec.lastElement() + "'"; -*/ - } //to outer while + + } + + for (Iterator iter = linesToDelete.iterator(); iter.hasNext();) + { + String id = (String) iter.next(); + delPSt.setString(1, id); + delPSt.execute(); + + } + + long endTime = System.currentTimeMillis(); + System.out.println("Time for incremental synchronize elapsed " + (endTime - startTime)); } // to idfield if } // table loop } catch (Exception e) { System.out.println("Error while connecting to database " + e); + e.printStackTrace(); if (dialog != null) { dialog.setVisible(false); @@ -2171,6 +2197,61 @@ class Convert dialog.setVisible(false); } + } + private static void performSynchronize( + String idField, + Vector vec, + String tempQuery, + TreeSet linesToDelete, + TreeSet linesToAppend, + PreparedStatement insPst, + PreparedStatement updPst, + PreparedStatement delPSt, + int deltaID, + String delimiter) + throws SQLException, ParseException + { + Vector[] vectors = bean.getQueryData(tempQuery, deltaID); + int idIndex = vectors[1].indexOf(idField); + // todo arraylist code has to be added + for (Iterator iter = vectors[0].iterator(); iter.hasNext();) + { + Vector line = (Vector) iter.next(); + Object lineIDIndex = line.get(idIndex); + if (linesToAppend.contains(lineIDIndex)) + System.out.println("line " + linesToAppend.contains(line.get(idIndex)) + " " + lineIDIndex); + if (linesToAppend.contains(lineIDIndex)) + { + for (int l = 0; l < line.size(); ++l) + { + Object obj = line.get(l); + if (obj instanceof ArrayList) + obj = formatFileMakerArray((List) obj, delimiter); + if (obj != null) + insPst.setString(l + 1, obj.toString()); + else + insPst.setNull(l + 1, Types.NULL); + } + insPst.execute(); + + } else // update + { + for (int l = 0; l < line.size(); ++l) + { + Object obj = line.get(l); + if (obj instanceof ArrayList) + obj = formatFileMakerArray((List) obj, delimiter); + if (obj != null) + updPst.setString(l + 1, obj.toString()); + else + updPst.setNull(l + 1, Types.NULL); + } + updPst.setString(line.size() + 1, line.get(idIndex).toString()); + updPst.execute(); + + } + } + } // to method /** @@ -3770,6 +3851,6 @@ class Convert //command.append(convertText((String) names.get(tbIndex))); command.append(beanDest.getQC()); command.append("WHERE " + idField + " = ?"); - return command; + return command; } } \ No newline at end of file