# HG changeset patch # User Zoe Hong # Date 1446476721 -3600 # Node ID 2e1662afc81c42fecae12195b413cc765956c74e # Parent 5520f941f6cb8463db20466a9e9022d98aacde19 new: add synchronization for files uploaded to LGDataverse diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/gazetteer/bo/LGFile.java --- a/src/main/java/de/mpiwg/gazetteer/bo/LGFile.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/gazetteer/bo/LGFile.java Mon Nov 02 16:05:21 2015 +0100 @@ -1,5 +1,6 @@ package de.mpiwg.gazetteer.bo; +import java.io.IOException; import java.io.Serializable; import javax.persistence.Column; @@ -11,6 +12,7 @@ import cl.maps.duplex.DuplexKey; import de.mpiwg.gazetteer.dataverse.DataverseUtils; +import de.mpiwg.gazetteer.utils.PropertiesUtils; @Entity @@ -47,10 +49,34 @@ @Transient private String content; - // if the file has been exported to DV, here is the datasetTitle + // if the file has been exported to LGDataverse, here is the datasetTitle @Column(name="datasetTitle") private String datasetTitle; + // if the file has been exported to LGDataverse, here is the datasetGlobalId, which is a string + @Column(name="datasetPersistentId") + private String datasetPersistentId; + + // if the file has been exported to LGDataverse, here is the fileId in LGDataverse; + // if it's null, means it hasn't been exported or has been deleted in LGDataverse. + @Column(name="fileIdInDv") + private Long fileIdInDv; + + + public String getDatasetUrl(){ + String dvnUrl = ""; + try { + dvnUrl = PropertiesUtils.getPropValue("dvn_server") + "/dataset.xhtml?persistentId="; + } catch (IOException e) { + e.printStackTrace(); + } + + if (this.getDatasetPersistentId() != null) { + return dvnUrl + getDatasetPersistentId() + "&version=DRAFT"; + } + + return dvnUrl; + } public Long getDvId() { return dvId; @@ -131,11 +157,7 @@ public DuplexKey getKey(){ return new DuplexKey(this.branchId, this.id); } - - @Override - public String toString(){ - return "LGFile [id=" + id + ", branchId=" + branchId + ", fileName=" + fileName + "]"; - } + public String getDatasetTitle() { return datasetTitle; @@ -146,4 +168,25 @@ } + + public Long getFileIdInDv() { + return fileIdInDv; + } + + public void setFileIdInDv(Long fileIdInDv) { + this.fileIdInDv = fileIdInDv; + } + + public String getDatasetPersistentId() { + return datasetPersistentId; + } + + public void setDatasetPersistentId(String datasetPersistentId) { + this.datasetPersistentId = datasetPersistentId; + } + + @Override + public String toString(){ + return "LGFile [id=" + id + ", branchId=" + branchId + ", fileName=" + fileName + "]"; + } } diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/gazetteer/dataverse/DataverseUtils.java --- a/src/main/java/de/mpiwg/gazetteer/dataverse/DataverseUtils.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/gazetteer/dataverse/DataverseUtils.java Mon Nov 02 16:05:21 2015 +0100 @@ -46,7 +46,7 @@ LGBranch branch = DataProvider.getInstance().getBranch(lgFile.getBranchId()); String link = PropertiesUtils.getPropValue("dvn_server") + "/saveResource"; - // get datasetGlobalId by datasetId + // --- get datasetGlobalId by datasetId String apiToken = PropertiesUtils.getPropValue("dvn_apitoken"); String dvnServerPath = PropertiesUtils.getPropValue("dvn_server"); @@ -68,6 +68,7 @@ } catch (Exception e) { e.printStackTrace(); } + // --- end getting datasetGlobalId MultipartUtility multipart = new MultipartUtility(link, "UTF-8"); @@ -85,21 +86,21 @@ // put table and name into a jsonObject then pass it JSONObject fileObj = new JSONObject(); - // TODO publish to LGDataverse using tab-delimiter data - //fileObj.put(branch.getLabel() + ".html", table); + // publish to LGDataverse using tab-delimiter data fileObj.put(branch.getLabel() + ".csv", table); - String fileStr = fileObj.toString(); multipart.addFormField("file", fileStr); - // end - String response = multipart.finish(); System.out.println(response); - return new JSONObject(response); + JSONObject resp = new JSONObject(response); + resp.put("datasetGlobalId", datasetGlobalId); // also return datasetGlobalId for file in branch + + return resp; + } public static JSONObject publishFile_old(Long fileId, String studyGlobalId, String userName, String password) throws Exception{ @@ -247,7 +248,7 @@ private static JSONObject getDatasetInfo(String datasetId) { JSONObject datasetJson = new JSONObject(); // datasetJson is the JSONObejct we collecting info about a dataset // show the dataset whose id is given: GET http://$SERVER/api/datasets/$id?key=$apiKey - + try { String dvnServerPath = PropertiesUtils.getPropValue("dvn_server"); @@ -596,6 +597,46 @@ return false; } + public static JSONObject getAllFilesInDataset(String datasetId) { + JSONObject filesJson = new JSONObject(); + + // api: GET http://$SERVER/api/datasets/$id/versions/$versionId/files?key=$apiKey + /* Lists all the file metadata, for the given dataset and version: + * version could be one of the follows: + * :draft the draft version, if any + * :latest either a draft (if exists) or the latest published version. + * :latest-published the latest published version + * x.y a specific version, where x is the major version number and y is the minor version number. + * x same as x.0 + */ + + HttpStringResponse response = null; + try { + String apiToken = PropertiesUtils.getPropValue("dvn_apitoken"); + String query = PropertiesUtils.getPropValue("dvn_server") + "/api/datasets/" + datasetId + "/versions/:draft/files?key=" + apiToken; + + response = HTTPUtils.getStringResponse(query); + JSONObject json = new JSONObject(response.content); + + if(StringUtils.equals(json.getString("status"), "OK")){ + return json; + } else { + filesJson.put("status", "Error"); + } + + } catch (Exception e) { + if(response != null){ + System.err.println(response.content); + } + e.printStackTrace(); + } + + + return filesJson; + + + } + } diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/gazetteer/utils/DBService.java --- a/src/main/java/de/mpiwg/gazetteer/utils/DBService.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/gazetteer/utils/DBService.java Mon Nov 02 16:05:21 2015 +0100 @@ -622,6 +622,20 @@ } + protected static List getLGFileInDatasetFromDB(String datasetPersistentId) { + List list = null; + + Session session = HibernateUtil.getSessionFactory().getCurrentSession(); + session.getTransaction().begin(); + Query query = session.createQuery("from LGFile where datasetPersistentId = :datasetPersistentId and fileIdInDv != NULL"); + query.setString("datasetPersistentId", datasetPersistentId); + list = query.list(); + session.getTransaction().commit(); + + + return list; + } + protected static List getAllLGFileFromDB(){ List list = null; diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/gazetteer/utils/DataProvider.java --- a/src/main/java/de/mpiwg/gazetteer/utils/DataProvider.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/gazetteer/utils/DataProvider.java Mon Nov 02 16:05:21 2015 +0100 @@ -24,6 +24,10 @@ return instance; } + public static void resetInstance() { + instance = null; + } + public DataProvider(){ logger.info("##### Starting DataProvider #####"); } diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/gazetteer/utils/UpdateFileStatus.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/mpiwg/gazetteer/utils/UpdateFileStatus.java Mon Nov 02 16:05:21 2015 +0100 @@ -0,0 +1,96 @@ +package de.mpiwg.gazetteer.utils; + + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.logging.Logger; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang.StringUtils; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import de.mpiwg.gazetteer.bo.LGFile; +import de.mpiwg.gazetteer.dataverse.DataverseUtils; + + +public class UpdateFileStatus extends HttpServlet{ + + + private static final Logger logger = Logger.getLogger(UpdateFileStatus.class.getName()); + + + public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + // Set response content type + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + + JSONObject jsonResponse = new JSONObject(); + + String datasetId = request.getParameter("datasetId"); + //logger.info("datasetId: " + datasetId); + + JSONObject filesJson = DataverseUtils.getAllFilesInDataset(datasetId); + + String datasetPersistentId = request.getParameter("datasetPersistentId"); + logger.info("sync files for dataset: " + datasetPersistentId); + + try { + + if(StringUtils.equals(filesJson.getString("status"), "OK")){ + // get all files with "datasetPersistentId=datasetPersistentId" from db + List filesInLGFile = DBService.getLGFileInDatasetFromDB(datasetPersistentId); + + //logger.info("files in dataset in LGServices: " + filesInLGFile); + //logger.info("files: "); + + // find files in the "filesInLGFile" but not in the "filesJson" + JSONArray data = filesJson.getJSONArray("data"); + + for (int i = 0; i < filesInLGFile.size(); i ++) { + Long fileIdInDv = filesInLGFile.get(i).getFileIdInDv(); + boolean updateFileIdInDv2Null = true; + + for (int j = 0 ; j < data.length(); j ++) { + int id = data.getJSONObject(j).getJSONObject("datafile").getInt("id"); + //logger.info("id= "+id + ", fileIdInDv= "+fileIdInDv); + + if (id == fileIdInDv) { + // found file, so don't update fileIdInDv + updateFileIdInDv2Null = false; + break; + } + } + + if (updateFileIdInDv2Null) { + // update file in LGServices database.File + logger.info("update fileIdInDv2=" + fileIdInDv + " for file (id): " + filesInLGFile.get(i).getId() ); + filesInLGFile.get(i).setFileIdInDv(null); + + Date date = new Date(); + DBService.saveDBEntry(filesInLGFile.get(i), date); + } + } + + jsonResponse.put("status", "OK"); + } + } catch (JSONException e1) { + e1.printStackTrace(); + } + + // ====================== + + response.setHeader("Access-Control-Allow-Origin", "*"); + + java.io.PrintWriter out = response.getWriter(); + out.print(jsonResponse); + out.flush(); + + } +} diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/web/branch/SortBranchByPublishedInDataverse.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/main/java/de/mpiwg/web/branch/SortBranchByPublishedInDataverse.java Mon Nov 02 16:05:21 2015 +0100 @@ -0,0 +1,21 @@ +package de.mpiwg.web.branch; + +import java.util.Comparator; + +import de.mpiwg.gazetteer.bo.LGBranch; + + +public class SortBranchByPublishedInDataverse implements Comparator{ + + public int compare(LGBranch o1, LGBranch o2) { + if(o1 == null || o2 == null){ + return o1.getId().compareTo(o2.getId()); + } + + String o1s = (o1.isPublished())? "1":"0"; + String o2s = (o2.isPublished())? "1":"0"; + + return o1s.compareTo(o2s); + } + +} \ No newline at end of file diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/web/jsp/BranchPage.java --- a/src/main/java/de/mpiwg/web/jsp/BranchPage.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/web/jsp/BranchPage.java Mon Nov 02 16:05:21 2015 +0100 @@ -60,15 +60,27 @@ if(resp.has("status") && StringUtils.equals(resp.getString("status"), "ok")){ // update record for the uploaded file LGFile file = DataProvider.getInstance().getFile(getFileId()); - long dvId = Long.parseLong(this.getDatasetId()); // dvId means datasetId, and all dataset are in "Chinese Local Gazetteers" dataverse + long dvId = Long.parseLong(this.getDatasetId()); // dvId means datasetId file.setDvId(dvId); + String datasetTitle = resp.getJSONObject("fileMetadata").getString("datasetTitle"); - file.setDatasetTitle(datasetTitle); + // get fileId in LGDataverse, so that we can associate the two files + // TODO keep track on them. If one in LGDataverse get deleted, notify the one in LGServices + + Long fileIdInDv = resp.getJSONObject("fileMetadata").getLong("fileId"); + System.out.println("fileId in LGDataverse: " + fileIdInDv); + file.setFileIdInDv(fileIdInDv); + + + // get datasetGlobalId and save it to file database table, so that we will have the url to the dataset + String datasetGlobalId = resp.getString("datasetGlobalId"); + file.setDatasetPersistentId(datasetGlobalId); // the datasetGlobalId here is persistentId + + // update to File table in mysql database DataProvider.getInstance().updateFile(file); - addMsg("The File has been uploaded to dataverse."); } else if (StringUtils.equals(resp.getString("status"), "error")) { @@ -123,6 +135,15 @@ */ + + public void forceReloadBranch() { + // reload DataProvider + DataProvider.resetInstance(); + + this.loadBranch(this.branch); + } + + public void loadBranch(String branchId0){ try { @@ -310,5 +331,7 @@ this.saveBranch0(); } + + } diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/web/jsp/HomePage.java --- a/src/main/java/de/mpiwg/web/jsp/HomePage.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/web/jsp/HomePage.java Mon Nov 02 16:05:21 2015 +0100 @@ -29,6 +29,7 @@ + import de.mpiwg.gazetteer.bo.LGBranch; import de.mpiwg.gazetteer.db.DBBook; import de.mpiwg.gazetteer.utils.DBService; @@ -39,6 +40,7 @@ import de.mpiwg.web.branch.SortBranchByLabel; import de.mpiwg.web.branch.SortBranchByLastModified; import de.mpiwg.web.branch.SortBranchByPeriod; +import de.mpiwg.web.branch.SortBranchByPublishedInDataverse; import de.mpiwg.web.branch.SortBranchBySectionName; public class HomePage extends AbstractJSPPage{ @@ -318,7 +320,16 @@ Collections.reverse(this.completeBranchList); filter(); } + public void sortByPublishedInDataverseUp() { + Collections.sort(this.completeBranchList, new SortBranchByPublishedInDataverse()); + filter(); + } + public void sortByPublishedInDataverseDown() { + Collections.sort(this.completeBranchList, new SortBranchByPublishedInDataverse()); + Collections.reverse(this.completeBranchList); + filter(); + } public int getBranchNumber() { return branchNumber; diff -r 5520f941f6cb -r 2e1662afc81c src/main/java/de/mpiwg/web/jsp/JSPProxy.java --- a/src/main/java/de/mpiwg/web/jsp/JSPProxy.java Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/java/de/mpiwg/web/jsp/JSPProxy.java Mon Nov 02 16:05:21 2015 +0100 @@ -41,8 +41,11 @@ getSessionBean().getBranchPage().publishFile(); }else if(StringUtils.equals(action, "updateLabel")){ getSessionBean().getBranchPage().updateLabel(); + }else if (StringUtils.equals(action, "forceReloadBranch")) { + getSessionBean().getBranchPage().forceReloadBranch(); } + return BranchPage.page; @@ -97,6 +100,10 @@ getSessionBean().getHomePage().sortByLastModifiedUp(); } else if(StringUtils.equals(action, "sortByLastModifiedDown")) { getSessionBean().getHomePage().sortByLastModifiedDown(); + } else if(StringUtils.equals(action, "sortByPublishedInDataverseUp")) { + getSessionBean().getHomePage().sortByPublishedInDataverseUp(); + } else if(StringUtils.equals(action, "sortByPublishedInDataverseDown")) { + getSessionBean().getHomePage().sortByPublishedInDataverseDown(); } return HomePage.page; diff -r 5520f941f6cb -r 2e1662afc81c src/main/resources/config.properties --- a/src/main/resources/config.properties Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/resources/config.properties Mon Nov 02 16:05:21 2015 +0100 @@ -4,7 +4,6 @@ #Local db_gazetter_username=root db_gazetter_password=root -#dvn_server=http://localhost/dvn dvn_server=http://localhost:8082 dvn_apitoken=9dd1f749-8c42-49ab-a2ba-fbb963c2ff90 dv_id=2 @@ -17,7 +16,9 @@ #localgazetteers-dev -> localgazetteers #db_gazetter_username=root #db_gazetter_password=admin -#dvn_server=http://localgazetteers.mpiwg-berlin.mpg.de/dvn +#dvn_server=http://dvn.localgazetteers.mpiwg-berlin.mpg.de +#dvn_apitoken= +#dv_id=1 #root_server=http://localgazetteers.mpiwg-berlin.mpg.de/LGServices #toc_interface=http://localgazetteers.mpiwg-berlin.mpg.de/LGToc #extraction_interface=http://localgazetteers.mpiwg-berlin.mpg.de/extraction-interface @@ -26,11 +27,10 @@ #localgazetteers -> localgazetteers-test #db_gazetter_username=root #db_gazetter_password=admin -#dvn_server=http://localgazetteers-test.mpiwg-berlin.mpg.de/dvn -#dvn_server=http://localgazetteers-test.mpiwg-berlin.mpg.de:8081 -#dvn_apitoken= +#dvn_server=http://dvn.localgazetteers-test.mpiwg-berlin.mpg.de +#dvn_apitoken=6acf9525-faba-4ea6-bfff-f83b44386c62 #dv_id=1 #root_server=http://localgazetteers-test.mpiwg-berlin.mpg.de/LGServices #toc_interface=http://localgazetteers-test.mpiwg-berlin.mpg.de/LGToc #extraction_interface=http://localgazetteers-test.mpiwg-berlin.mpg.de/extraction-interface -#localgazetteers_dvId= \ No newline at end of file +#localgazetteers_dvId=2 diff -r 5520f941f6cb -r 2e1662afc81c src/main/webapp/WEB-INF/web.xml --- a/src/main/webapp/WEB-INF/web.xml Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/webapp/WEB-INF/web.xml Mon Nov 02 16:05:21 2015 +0100 @@ -46,5 +46,13 @@ 120 - + + + de.mpiwg.gazetteer.utils.UpdateFileStatus + /updateFileStatus + + + de.mpiwg.gazetteer.utils.UpdateFileStatus + de.mpiwg.gazetteer.utils.UpdateFileStatus + diff -r 5520f941f6cb -r 2e1662afc81c src/main/webapp/componentes/template.jsp --- a/src/main/webapp/componentes/template.jsp Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/webapp/componentes/template.jsp Mon Nov 02 16:05:21 2015 +0100 @@ -62,7 +62,7 @@
- Home + Your Branch Create File Search Books diff -r 5520f941f6cb -r 2e1662afc81c src/main/webapp/pages/branchPage.jsp --- a/src/main/webapp/pages/branchPage.jsp Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/webapp/pages/branchPage.jsp Mon Nov 02 16:05:21 2015 +0100 @@ -71,10 +71,19 @@ <% } else { %> - - - <% if(sessionBean.getBranchPage().getBranch() != null) { %> +
Branch Details +
+ + + +
+ +
+
- + @@ -284,7 +293,7 @@ - + @@ -308,7 +317,7 @@ - + @@ -320,10 +329,14 @@ <% if(file.getDvId() == null) {%> - - <% } else { %> - - + + <% } else if (file.getFileIdInDv() == null) { %> + <%= file.getDatasetTitle() %> + + + <% } else { %> + + <% } %> diff -r 5520f941f6cb -r 2e1662afc81c src/main/webapp/pages/home.jsp --- a/src/main/webapp/pages/home.jsp Fri Sep 25 14:32:47 2015 +0200 +++ b/src/main/webapp/pages/home.jsp Mon Nov 02 16:05:21 2015 +0100 @@ -42,7 +42,7 @@ action="<%=sessionBean.getApplicationBean().getRootServer()%>/proxy.jsp" > -
User's Branches +
Your Branch List @@ -224,7 +224,7 @@ - + - + + + + + + + +
+ + + +
+ +
+ +
+
+ + + @@ -293,7 +316,7 @@ - <%=branch.isPublished() %> + <%= (branch.isPublished()) ? "V":"X" %>