--- FM2SQL/Attic/Convert.java 2004/06/21 11:31:35 1.58 +++ FM2SQL/Attic/Convert.java 2004/07/27 10:41:36 1.63 @@ -19,10 +19,12 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.UnsupportedEncodingException; +import java.net.URL; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; @@ -65,8 +67,6 @@ class Convert System.out.println("Usage: java Convert "); System.exit(-1); } - if (!(new File(args[0]).exists())) - System.exit(0); try { file = new FileOutputStream("./log.txt"); @@ -75,9 +75,13 @@ class Convert e1.printStackTrace(); } PrintStream stream = new PrintStream(file); - System.setOut(stream); - System.setErr(stream); + // System.setOut(stream); + //System.setErr(stream); + readXMLFile(args[0]); + if (!(new File(args[0]).exists())) + System.exit(0); + System.out.println("Finished!"); //convert("jdbc:fmpro:http://141.14.237.74:8050","jdbc:postgresql://erebos/test",null,null); } @@ -1218,6 +1222,8 @@ class Convert Node node3 = root.find("convert/source/database", new int[] { 1, 1, i }); Node nodeMode = root.find("convert/source/database/mode", new int[] { 1, 1, i, 1, 1 }); Node delimiterNode = root.find("convert/source/database/delimiter", new int[] { 1, 1, i, 1, 1 }); + Node useNormanToUnicodeMapper = root.find("convert/source/database/usenormantounicodemapper", new int[] { 1, 1, i, 1, 1 }); + if (node3 == null) throw new Error("parse error database tag missing"); if (node == null) @@ -1228,6 +1234,12 @@ class Convert throw new Error("parse error password tag missing"); if (delimiterNode != null) delimiter = delimiterNode.getCharacters(); + if(useNormanToUnicodeMapper!=null) + { + database.setUseNormanToUnicodeMapper(Boolean.valueOf(useNormanToUnicodeMapper.getCharacters()).booleanValue()); + System.out.println("useMapper "+Boolean.valueOf(useNormanToUnicodeMapper.getCharacters().trim()).booleanValue()); + } + String url = node.getCharacters(); String user = node1.getCharacters(); String password = node2.getCharacters(); @@ -1249,6 +1261,8 @@ class Convert mode = DataBase.UPDATE_MODE; else if (modeString.equals("delete")) mode = DataBase.DELETE_MODE; + else if (modeString.equals("synchronize")) + mode = DataBase.SYNCHRONIZE_MODE; // if(node3!=null) // System.out.println(node3.name); @@ -1317,11 +1331,20 @@ class Convert for (Iterator iter = databases.iterator(); iter.hasNext();) { DataBase db = (DataBase) iter.next(); - if (mode != DataBase.UPDATE_MODE) + if (mode == DataBase.CONVERT_MODE || mode == DataBase.APPEND_MODE) convertBatch(db.bean, database, db.tables, db.layouts, db.selects, db.creates, db.ids, mode, db.delimiter); else - update(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode); - + if (mode == DataBase.UPDATE_MODE) + update(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode); + else if(mode==DataBase.SYNCHRONIZE_MODE) + { + user =bean.user; + passwd =bean.passwd; + userDest =database.user; + passwdDest =database.passwd; + + synchronize(db.bean.url, database.url, db.tables, db.layouts, db.selects, db.creates, db.ids, mode); + } } // printContents(node3); // FM2SQL.fmInstance=new FM2SQL(); @@ -1391,9 +1414,16 @@ class Convert Node node3 = root.find("convert/source/database", new int[] { 1, 1, i }); Node nodeMode = root.find("convert/source/database/mode", new int[] { 1, 1, i, 1, 1 }); Node delimiterNode = root.find("convert/source/database/delimiter", new int[] { 1, 1, i, 1, 1 }); + Node useNormanToUnicodeMapper = root.find("convert/source/database/usenormantounicodemapper", new int[] { 1, 1, i, 1, 1 }); if (delimiterNode != null) delimiter = delimiterNode.getCharacters(); + if(useNormanToUnicodeMapper!=null) + { + database.setUseNormanToUnicodeMapper(Boolean.valueOf(useNormanToUnicodeMapper.getCharacters()).booleanValue()); + System.out.println("useMapper "+Boolean.valueOf(useNormanToUnicodeMapper.getCharacters().trim()).booleanValue()); + } + if (node3 == null) throw new Error("parse error database tag missing"); if (node == null) @@ -1548,9 +1578,17 @@ class Convert { try { - // read XML Metadata from a file - FileInputStream fi = new FileInputStream(xmlFile); - InputStreamReader isr = new InputStreamReader(fi, "UTF-8"); + InputStream stream = null; + + if(xmlFile.indexOf("file://")>=0||xmlFile.indexOf("http://")>=0) + { + URL url=new URL(xmlFile); + stream = url.openStream(); + } + else + // read XML Metadata from a file + stream = new FileInputStream(xmlFile); + InputStreamReader isr = new InputStreamReader(stream, "UTF-8"); BufferedReader buffr = new BufferedReader(isr); StringBuffer sb = new StringBuffer(); int c = 0; @@ -1582,11 +1620,14 @@ class Convert Vector tables; Vector ids; String delimiter = "//"; + boolean useNormanToUnicodeMapper = false; + final static int CONVERT_MODE = 1; final static int APPEND_MODE = 2; final static int UPDATE_MODE = 3; final static int DELETE_MODE = 4; - + final static int SYNCHRONIZE_MODE = 5; + int mode = -1; public DataBase(DBBean bean, Vector tables, Vector layouts, Vector selects, Vector creates, Vector ids, int mode) @@ -1624,6 +1665,8 @@ class Convert modeString = "delete"; buffr.write(" " + modeString + "\n"); + buffr.write(" " + useNormanToUnicodeMapper + "\n"); + int index = 0; while (index < tables.size()) { @@ -1884,6 +1927,246 @@ class Convert } } // to method + + /** + * synchronize method based on delete method code + * + * @param source + * @param destination + * @param names + * @param layouts + * @param selects + * @param creates + * @param ids + * @param mode + * @throws Exception + */ + // TODO implement append,update and delete in one method + // TODO using id based algorithm + public static void synchronize(String source, String destination, Vector names, Vector layouts, Vector selects, Vector creates, Vector ids, int mode) throws Exception + { + System.out.println(" bin in synchronize!!!"); + FM2SQL.ProgressDialog dialog = null; + if (FM2SQL.fmInstance != null) + { + dialog = new FM2SQL.ProgressDialog(FM2SQL.fmInstance, bean); + dialog.setTitle("Synchronize running ..."); + dialog.title.setText("Getting table data ..."); + dialog.setLocation(FM2SQL.fmInstance.getLocationOnScreen().x + (FM2SQL.fmInstance.getWidth() - 400) / 2, FM2SQL.fmInstance.getLocationOnScreen().y + (FM2SQL.fmInstance.getHeight() - 250) / 2); + dialog.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + FM2SQL.fmInstance.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)); + dialog.thread = Thread.currentThread(); + } + // setting user and passwd + bean.setUserAndPasswd(user, passwd); + // setting user and passwd + beanDest.setUserAndPasswd(userDest, passwdDest); + if (dialog != null) + dialog.setSize(400, 250); + StringBuffer command = null; + String query = null; + try + { + //bean.setConnection("jdbc:fmpro:http://141.14.237.74:8050"); + //bean.setConnection("jdbc:postgresql://erebos/test","postgres","rogo"); + bean.setConnection(source); + if (names == null) + names = bean.getTableNames(); + // Collections.sort(names); + int tbIndex = 1; + + // System.out.println("Start at "+names.indexOf("archimedes_facsimiles")); + for (tbIndex = 0; tbIndex < names.size(); ++tbIndex) + { + Vector[] result = null; + java.util.TreeSet myIds = new TreeSet(); + java.util.TreeSet myIdsDest = new TreeSet(); + int deltaID = 1; + String idField = ""; + String destTableName = ""; + + try + { + query = "select * from " + bean.getQC() + names.get(tbIndex).toString() + bean.getQC(); + String layout = (layouts.isEmpty()) ? "" : layouts.get(tbIndex).toString(); + query = (selects != null) ? selects.get(tbIndex).toString() : query; + //if vectors[1].get(i) != null) + if (!layout.equals("")) + { + layout = " layout " + bean.getQC() + layout + bean.getQC(); + String name = names.get(tbIndex).toString(); + StringBuffer queryLayout = new StringBuffer(query); + queryLayout.insert(queryLayout.indexOf(name) + name.length() + 1, " " + layout); + query = queryLayout.toString(); + System.out.println("added layout " + query); + + } + if(dialog!=null) { + dialog.title.setText("Getting table data ..."); + dialog.table.setText(names.get(tbIndex).toString()); + dialog.status.setText("Table " + (tbIndex + 1) + " of " + names.size()); + dialog.show(); + } + bean.getConnection(); + bean.makeQuery(query, 50); + idField = ids.get(tbIndex).toString(); + + } catch (Exception e) + { + continue; + } + // determine destTableName from createStatement or from source table name + if (!creates.get(tbIndex).equals("")) + { + String create = creates.get(tbIndex).toString().toLowerCase(); + int fromIndex = create.indexOf("table") + 5; + int toIndex = create.indexOf("("); + destTableName = create.substring(fromIndex, toIndex).replaceAll(beanDest.getQC(), "").trim(); + System.out.println("destTable " + destTableName); + + } else + destTableName = convertText(names.get(tbIndex).toString()); + + // for id kram + Vector vec = null; + Vector vecDest = null; + // tempo + beanDest.setConnection(destination); + int rowCount = (idField != "") ? myIds.size() : bean.getRowCount(query); + String tempID = bean.getQC() + idField + bean.getQC(); + String tempIDdest = beanDest.getQC() + convertText(idField) + beanDest.getQC(); + + int endIndex = -1; + String tempQuery = query; + String destQuery = query.replaceAll(names.get(tbIndex).toString(), destTableName); + String tempQueryDest = destQuery; + // remove extra query parts destQuery.substring(0,destQuery.lastIndexOf(destTableName)+destTableName.length()+1); + System.out.println("new Query " + tempQueryDest); + if (!idField.equals("")) + { + long startTime = System.currentTimeMillis(); + int counter = -1; + while (true) + { + ++counter; + if (counter == 0 && dialog != null) + dialog.title.setText("Check if data is available"); + else if (dialog != null) + dialog.title.setText("Check if more data is available"); + myIds = bean.getIDVector(ids.get(tbIndex).toString(), (String) names.get(tbIndex), tempQuery, numHits); + myIdsDest = beanDest.getIDVector(convertText(idField), destTableName, tempQueryDest, numHits); + if (myIds.isEmpty()) + break; + vec = new Vector(myIds); + vecDest = new Vector(myIdsDest); + rowCount = vec.size(); + // Deletion will work this way + Vector deleted = new Vector(vec); + Vector linesToDelete = new Vector(vecDest); + + Vector linesToAppend = new Vector(vec); + // remove all lines that are already in dest database + linesToAppend.removeAll(vecDest); + // remove all lines that should not be deleted + linesToDelete.removeAll(deleted); + System.out.println("linesToAppend "+linesToAppend+" "+vecDest+" "+destTableName); + System.out.println("linesToDelete "+linesToDelete+" "+vecDest+" "+destTableName); + // prepare new query for next chunk + if (query.indexOf("where") > 0) + tempQuery = query + " and " + tempID + ">='" + vec.firstElement() + "' and "+tempID + "<='" + vec.lastElement() + "' order by "+tempID; + else + tempQuery = query + " where " + tempID + ">='" + vec.firstElement() + "' and "+tempID + "<='" + vec.lastElement() + "'";// order by "+tempID; + System.out.println("tempQuery is now "+tempQuery+" order by "+tempID); + // bean.makeQuery(tempQuery,vec.size()); + Vector[] vectors=bean.getQueryData(tempQuery,vec.size()); + System.out.println(vectors[0]); + System.out.println( bean.getColumnNames().indexOf(idField)); + System.exit(0); + // System.out.println("ID LIST SIZE " + Math.round((double) myIds.size() / (double) numIntervalls) + " " + myIdsDest.size()); + /// @TODO complete delete task remove query show lines to be deleted let user choose if he wants that + System.out.println("number of lines to be deleted " + linesToDelete.size()); + deltaID = (int) Math.round((double) myIds.size() / (double) numIntervalls); + beanDest.setConnection(destination); + + Statement stm = beanDest.getConnection().createStatement(); + + Vector tables = beanDest.getTableNames(); + // Collections.sort(tables); + System.out.println(names.get(tbIndex) + " " + tables.indexOf(convertText((String) names.get(tbIndex)))); // "//beanDest.getTypeNames()); + tables = beanDest.getTableNames(); + // System.out.println(beanDest.getTableNames(beanDest.getCatalogs().get(2).toString())); + stm = beanDest.getConnection().createStatement(); + + if (dialog != null) + dialog.title.setText(" Deleting table data ..."); + + int j = -1; + + Vector row = null; + command = new StringBuffer(); + + command.append("DELETE FROM"); + command.append(beanDest.getQC()); + command.append(destTableName); + //command.append(convertText((String) names.get(tbIndex))); + command.append(beanDest.getQC()); + int size = bean.getColumnNames().size(); + command.append("WHERE " + convertText(ids.get(tbIndex).toString()) + " = ?"); + PreparedStatement pstm = beanDest.getConnection().prepareStatement(command.toString()); + System.out.println(command + " " + tbIndex); + //int rowCount = bean.getRowCount(query); + // int idIndex = bean.getColumnNames().indexOf(ids.get(tbIndex)); + while (true) + { + + ++j; + if (j == linesToDelete.size()) + break; + //print rows + pstm.setString(1, linesToDelete.get(j).toString()); + System.out.println(pstm.toString()); + pstm.execute(); + if (dialog != null) + dialog.progress.setValue((int) (((double) (j + 1) / (double) rowCount) * 100.0)); + command = null; + } + // prepare new query for next chunk + if (query.indexOf("where") > 0) + tempQuery = query + " and " + tempID + ">'" + vec.lastElement() + "'"; + else + tempQuery = query + " where " + tempID + ">'" + vec.lastElement() + "'"; + + } //to outer while + } // to idfield if + } // table loop + + } catch (Exception e) + { + System.out.println("Error while connecting to database " + e); + if (dialog != null) + { + dialog.setVisible(false); + dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + FM2SQL.fmInstance.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + } + java.io.ByteArrayOutputStream b = new java.io.ByteArrayOutputStream(); + java.io.PrintStream stream = new java.io.PrintStream(b); + stream.print(command + "\n\n"); + e.printStackTrace(stream); + FM2SQL.showErrorDialog(b.toString(), "Error occured !"); + + } + if (dialog != null) + { + dialog.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + FM2SQL.fmInstance.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR)); + + dialog.setVisible(false); + } + } // to method + + + /** * Converts input String in norman encoding to unicode * @param inp