Binary file docs/LGServices_Developer_Guide.odt has changed

--- a/src/main/java/de/mpiwg/gazetteer/bo/LGBranch.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/bo/LGBranch.java	Fri Sep 11 10:34:25 2015 +0200
@@ -64,10 +64,43 @@
 	@Transient
 	private boolean published = false;
 	
+	public boolean isEmpty() {
+		if (this.section == null) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+	
+	@Transient
+	private boolean sectionDeprecated = false;
+	
+	public boolean isDeprecated() {
+		if (this.sectionDeprecated) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+	
 	public void loadTransientData(){
 		try {
 			this.section = DBService.getSectionWithContent(sectionId);
-			this.book = section.getBook();
+			if (this.section == null) {
+				// the section has been deleted by user when doing TOC editing.
+				
+				this.sectionDeprecated = true;
+				// get deprecated section from sections_revisions table
+				this.section = DBService.getDeprecatedSectionWithContent(sectionId);
+				if (this.section == null) {
+					System.out.println("section " + sectionId + " doesn't exist.");
+					return;
+				} else {
+					System.out.println("find deprecated section " + sectionId + " in sections_revisions table.");
+				}
+				
+			} 
+			this.book = section.getBook();	
 			this.transientDataLoaded = true;
 			this.published = atLeastOneFilePublished();
 		} catch (SQLException e) {

--- a/src/main/java/de/mpiwg/gazetteer/bo/LGFile.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/bo/LGFile.java	Fri Sep 11 10:34:25 2015 +0200
@@ -24,7 +24,7 @@
 	@Column(name="branchId")
 	private Long branchId;
 	
-	//if this file has been exported to DV, here is saved the id.
+	//if this file has been exported to DV, here is saved the id (for dataset)
 	@Column(name="dvId")
 	private Long dvId;
 	
@@ -47,6 +47,11 @@
 	@Transient
 	private String content;
 	
+	// if the file has been exported to DV, here is the datasetTitle
+	@Column(name="datasetTitle")
+	private String datasetTitle;
+	
+	
 	public Long getDvId() {
 		return dvId;
 	}
@@ -131,4 +136,14 @@
 	public String toString(){
 		return "LGFile [id=" + id + ", branchId=" + branchId + ", fileName=" + fileName + "]";
 	}
+
+	public String getDatasetTitle() {
+		return datasetTitle;
+	}
+
+	public void setDatasetTitle(String datasetTitle) {
+		this.datasetTitle = datasetTitle;
+	}
+
+	
 }

--- a/src/main/java/de/mpiwg/gazetteer/dataverse/DataverseUtils.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/dataverse/DataverseUtils.java	Fri Sep 11 10:34:25 2015 +0200
@@ -2,6 +2,8 @@
 
 
 import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -12,6 +14,8 @@
 
 import de.mpiwg.gazetteer.bo.LGBranch;
 import de.mpiwg.gazetteer.bo.LGFile;
+import de.mpiwg.gazetteer.dataverse.bo.Datasets;
+import de.mpiwg.gazetteer.dataverse.bo.Dataverse;
 import de.mpiwg.gazetteer.dataverse.bo.Study;
 import de.mpiwg.gazetteer.dataverse.bo.VDCUser;
 import de.mpiwg.gazetteer.utils.DataProvider;
@@ -21,6 +25,8 @@
 
 public class DataverseUtils {
 	
+	
+	
 	private static List<VDCUser> userList;
 	
 	static{
@@ -29,36 +35,65 @@
 		userList.add(new VDCUser(new Long(11), "john", "1234"));
 		userList.add(new VDCUser(new Long(12), "zoe", "1234"));
 		userList.add(new VDCUser(new Long(13), "paul", "1234"));
+		
 	}
-	
+		
 
-	/**
-	 * 
-	 *  curl --form upload=@pom.xml --form user=networkAdmin --form password=networkAdmin --form studyId=hdl:TEST/10002 http://localhost/dvn/saveResource
-	 * see: http://www.codejava.net/java-se/networking/upload-files-by-sending-multipart-request-programmatically
-	 * @param branchId
-	 * @param studyGlobalId
-	 * @throws Exception
-	 */
-		public static JSONObject publishFile(Long fileId, String studyGlobalId, String userName, String password) throws Exception{
+	
+	public static JSONObject publishFile(Long fileId, String datasetId, String userName, String password) throws Exception{
 			
 			LGFile lgFile = DataProvider.getInstance().getFile(fileId);
 			LGBranch branch = DataProvider.getInstance().getBranch(lgFile.getBranchId());
 			String link = PropertiesUtils.getPropValue("dvn_server") + "/saveResource";
 			
+			// get datasetGlobalId by datasetId
+			String apiToken = PropertiesUtils.getPropValue("dvn_apitoken");
+			String dvnServerPath = PropertiesUtils.getPropValue("dvn_server");
+		
+			String query2 = dvnServerPath + "/api/datasets/"+ datasetId + "?key=" + apiToken;
+			HttpStringResponse response1 = null;
+			String datasetGlobalId = ""; 
+			try {
+				response1 = HTTPUtils.getStringResponse(query2);
+				JSONObject jsonObject = new JSONObject(response1.content);
+				if(StringUtils.equals(jsonObject.getString("status"), "OK")){
+					JSONObject data = jsonObject.getJSONObject("data");
+					String dsProtocol = data.getString("protocol");
+					String dsAuthority = data.getString("authority");
+					String dsIdentifier = data.getString("identifier");
+					// concatenate info to datasetGlobalId
+					datasetGlobalId = dsProtocol + ":"+ dsAuthority + "/" + dsIdentifier;
+				} 
+				
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
 			
+	
 			MultipartUtility multipart = new MultipartUtility(link, "UTF-8");
-            
-            multipart.addFormField("studyId", studyGlobalId);
+            multipart.addFormField("datasetGlobalId", datasetGlobalId);
             multipart.addFormField("password", password);
             multipart.addFormField("user", userName);
-            //multipart.addFormField("fileLabel", branch.getLabel() + ".html");
-             
- 
-            String table = HTTPUtils.getTableOfFile(lgFile.getId());
+            
+            //String table = HTTPUtils.getTableOfFile(lgFile.getId());
+            String table = HTTPUtils.getTabDelimitedOfFile(lgFile.getId());
+            
             System.out.println("Table string lenght" + table.length());
             
-            multipart.addFilePart0(branch.getLabel() + ".html", table);
+            //multipart.addFilePart0(branch.getLabel() + ".html", table);
+            
+            // put table and name into a jsonObject then pass it
+            JSONObject fileObj = new JSONObject();
+            
+            // TODO publish to LGDataverse using tab-delimiter data 
+            //fileObj.put(branch.getLabel() + ".html", table);
+            fileObj.put(branch.getLabel() + ".csv", table);
+            
+            String fileStr = fileObj.toString();
+            multipart.addFormField("file", fileStr);
+            
+            // end
+            
             
             String response = multipart.finish();
 			
@@ -67,15 +102,291 @@
             return new JSONObject(response);
 	}
 	
-	public static List<Study> getStudies(String userName, String password) throws IOException{
-		List<Study> list = new ArrayList<Study>();
+	public static JSONObject publishFile_old(Long fileId, String studyGlobalId, String userName, String password) throws Exception{
+		
+		LGFile lgFile = DataProvider.getInstance().getFile(fileId);
+		LGBranch branch = DataProvider.getInstance().getBranch(lgFile.getBranchId());
+		String link = PropertiesUtils.getPropValue("dvn_server") + "/saveResource";
+		
+		
+		MultipartUtility multipart = new MultipartUtility(link, "UTF-8");
+        
+        multipart.addFormField("studyId", studyGlobalId);
+        multipart.addFormField("password", password);
+        multipart.addFormField("user", userName);
+        //multipart.addFormField("fileLabel", branch.getLabel() + ".html");
+  
+
+        String table = HTTPUtils.getTableOfFile(lgFile.getId());
+        System.out.println("Table string lenght" + table.length());
+        
+        multipart.addFilePart0(branch.getLabel() + ".html", table);
+        
+        String response = multipart.finish();
+		
+        System.out.println(response);
+		
+        return new JSONObject(response);
+	}
+	
+	
+	public static List<Datasets> getDatasetsInAllDataverse(String userName, String password){
+		// get the list of all dataverse
+		List<Datasets> datasetList = new ArrayList<Datasets>();
 		
-		String query = PropertiesUtils.getPropValue("dvn_server") + "/getStudies?user=" + userName + "&password=" + password;
+		try {
+			JSONArray dataverseArray;
+			dataverseArray = getAllDataverseAlias();
+			for (int i = 0; i < dataverseArray.length(); i++) {
+				String aAlias = dataverseArray.getString(i);
+				if (!StringUtils.equals("root", aAlias)) {					
+					// it is not "root" dataverse
+					List<Datasets> aDatasetList = getDatasets(userName, password, aAlias);
+
+					// combine datasetList into a List<Datasets> 
+					datasetList.addAll(aDatasetList);
+			
+				}
+			}
+			
+		} catch (KeyManagementException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (NoSuchAlgorithmException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (JSONException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+	
+		return datasetList;
+				
+	}
+	
+	
+	private static List<Datasets> getDatasets(String userName, String password, String dvId) throws IOException, JSONException, KeyManagementException, NoSuchAlgorithmException{
+		
+		String apiToken = PropertiesUtils.getPropValue("dvn_apitoken");
+		String dvnServerPath = PropertiesUtils.getPropValue("dvn_server");
+		
+		/* 
+		// list all the DvObjects under dataverse id. GET http://$SERVER/api/dataverses/$id/contents
+		String dvId = PropertiesUtils.getPropValue("dv_id");	// get Local gazetteers dataverse id from config file
+		// TODO the dvId above probably need to revise. Each user may have h/her own dataverse.
+		// In this case, need to find out which dvIds belong to the user.
+		
+		// in the position of 'dvId', it can be used as the alias also as recommended by dataverse
+		*/
+		String query1 = dvnServerPath + "/api/dataverses/"+ dvId + "/contents?key=" + apiToken;
+		
+		
+		List<Datasets> datasetList = new ArrayList<Datasets>();
 		
 		HttpStringResponse response = null;
 		try {
-			response = HTTPUtils.getStringResponse(query);
+			response = HTTPUtils.getStringResponse(query1);
 			JSONObject json = new JSONObject(response.content);
+
+			System.out.println("get contents in dataverse " + dvId + ": " + json);
+			
+			if(StringUtils.equals(json.getString("status"), "OK")){
+				// go through all datasets in this dataverse
+				JSONArray jsonData = json.getJSONArray("data");
+				for (int i = 0; i < jsonData.length(); i ++) {
+					JSONObject aDataset = jsonData.getJSONObject(i);
+					String datasetId = aDataset.getString("id");
+					
+					System.out.println("checking datasetId= " + datasetId);
+					
+					// check if the user has roles to access to the dataset. 
+					// needed to be contributor or above, or in defined groups
+					if (canPublish(userName, password, dvId, datasetId)) {
+						System.out.println("User @" + userName + " can upload files to datasetId= " + datasetId);
+						
+						JSONObject datasetJson = getDatasetInfo(datasetId);
+						
+						datasetJson.put("dvIdInDataverse", dvId);
+							
+						Datasets datasets = new Datasets(datasetJson);
+						datasetList.add(datasets);
+						
+					}
+					
+				}
+			}
+			
+		} catch (Exception e) {
+			if(response != null){
+				System.err.println(response.content);
+			}
+			e.printStackTrace();
+		}
+		return datasetList;
+	}
+	
+	
+	private static JSONArray getAllDataverseAlias() throws IOException, KeyManagementException, NoSuchAlgorithmException, JSONException {
+		
+		String dvnServerPath = PropertiesUtils.getPropValue("dvn_server");
+		String query = dvnServerPath + "/getAllDataverseAlias";		// get all dataverse alias from LGDataverse
+		
+		HttpStringResponse response = null;
+		response = HTTPUtils.getStringResponse(query);
+		JSONObject data = new JSONObject(response.content);
+		JSONArray allDataverseAlias = data.getJSONArray("dataverseAlias");
+			
+		return allDataverseAlias;
+		
+	}
+	
+	private static JSONObject getDatasetInfo(String datasetId) {
+		JSONObject datasetJson = new JSONObject();	// datasetJson is the JSONObejct we collecting info about a dataset
+		// show the dataset whose id is given: GET http://$SERVER/api/datasets/$id?key=$apiKey
+	
+		try {
+			
+			String dvnServerPath = PropertiesUtils.getPropValue("dvn_server");
+			String apiToken = PropertiesUtils.getPropValue("dvn_apitoken");
+			
+			String query2 = dvnServerPath+ "/api/datasets/"+ datasetId + "?key=" + apiToken;
+			HttpStringResponse response1 = null;
+			
+			
+			response1 = HTTPUtils.getStringResponse(query2);
+			JSONObject jsonObject = new JSONObject(response1.content);
+			if(StringUtils.equals(jsonObject.getString("status"), "OK")){			
+				// collect information for the dataset
+				datasetJson.put("status", "OK");
+			
+				JSONObject data = jsonObject.getJSONObject("data");
+				datasetJson.put("id", datasetId);
+			
+				datasetJson.put("authority", data.getString("authority"));
+				datasetJson.put("persistentURL", data.getString("persistentUrl"));
+				datasetJson.put("protocol", data.getString("protocol"));
+				datasetJson.put("identifier", data.getString("identifier"));
+	
+				JSONObject latestVersion = data.getJSONObject("latestVersion");
+				datasetJson.put("createTime", latestVersion.getString("createTime"));
+			
+				String versionState = latestVersion.getString("versionState");
+				datasetJson.put("versionState",versionState);
+			
+				if (StringUtils.equals(versionState, "RELEASED")) {
+					datasetJson.put("version", latestVersion.getString("versionNumber")+"."+latestVersion.getString("versionMinorNumber"));
+				} else {
+					datasetJson.put("version","");
+				}
+		
+				JSONArray citationFields = latestVersion.getJSONObject("metadataBlocks").getJSONObject("citation")
+					.getJSONArray("fields");
+				for (int j = 0; j < citationFields.length(); j++) {
+					JSONObject a = citationFields.getJSONObject(j);
+					if( StringUtils.equals(a.getString("typeName"),"title")) {
+						String title = a.getString("value");
+						//System.out.println("title: " + title);
+						datasetJson.put("title", title);
+					}
+				}
+
+			} else {
+				datasetJson.put("status", "ERROR");
+			
+			}
+		
+		
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	
+		
+		return datasetJson;
+		
+		
+	}
+
+	private static boolean canPublish(String userName, String password, String dvId, String datasetId) {
+		
+		String queryRoleAssignments;
+		try {
+			queryRoleAssignments = PropertiesUtils.getPropValue("dvn_server") + "/getRoleAssignments?user=" + userName + "&password=" + password 
+						+ "&datasetId=" + datasetId;
+
+			HttpStringResponse responseRoleAssignments = null;
+			
+			responseRoleAssignments = HTTPUtils.getStringResponse(queryRoleAssignments);
+			JSONObject raJson = new JSONObject(responseRoleAssignments.content);
+	
+			if(StringUtils.equals(raJson.getString("state"), "ok") && raJson.has("roleAssignments")){
+				JSONArray roleAssignments = raJson.getJSONArray("roleAssignments");
+				
+				for( int j = 0, size = roleAssignments.length(); j < size; j++ ) {
+					
+					JSONObject aRoleAssignment = roleAssignments.getJSONObject(j);
+					String aIdentifier = aRoleAssignment.getString("identifier");
+					String aRoleName = aRoleAssignment.getString("roleName");
+					
+				
+					boolean checkRoles = false;
+					// check user has the right to upload file:
+					// 1. check user identifier 
+					if (StringUtils.equals(aIdentifier, "@"+userName) ) {
+						checkRoles = true;
+					
+					// 2. check user in the group which has the right to upload file	
+					} else if (StringUtils.startsWith(aIdentifier, "&")) {
+						
+						String delims = "[&/-]+";
+						String[] tokens = aIdentifier.split(delims);
+						String group = tokens[tokens.length-1]; 	// parse group name from it. It is the last one.
+							
+						List<String> userGroups = new ArrayList<String>();	
+						userGroups.add(group);
+						// userGroups is designed to be List. But as here it is used as only one element, this could be modified in the future.
+						if (isUserInGroups(userName, userGroups) ) {
+							checkRoles = true;
+						}
+						
+					}
+				
+					if (checkRoles) {
+						if (StringUtils.equals(aRoleName, "Contributor") || StringUtils.equals(aRoleName, "Curator") || StringUtils.equals(aRoleName, "Admin")) {	
+							return true;
+						}
+					}
+						
+				}
+			
+			}
+
+		
+		} catch (IOException e) {
+			e.printStackTrace();
+		} catch (KeyManagementException e) {
+			e.printStackTrace();
+		} catch (NoSuchAlgorithmException e) {
+			e.printStackTrace();
+		} catch (JSONException e) {
+			e.printStackTrace();
+		}
+		return false;
+	}
+
+	// getStudy() has been replaced by getDatasets() when we migrate to dataverse 4.0
+	public static List<Study> getStudy(String userName, String password) throws IOException{
+	
+		String query = PropertiesUtils.getPropValue("dvn_server") + "/getStudies?user=" + userName + "&password=" + password;
+		List<Study> list = new ArrayList<Study>();
+		
+		HttpStringResponse response1 = null;
+		try {
+			response1 = HTTPUtils.getStringResponse(query);
+			JSONObject json = new JSONObject(response1.content);
 			if(json.has("studies")){
 				
 				JSONArray userArray = json.getJSONArray("studies");
@@ -87,8 +398,8 @@
 				}	
 			}
 		} catch (Exception e) {
-			if(response != null){
-				System.err.println(response.content);
+			if(response1 != null){
+				System.err.println(response1.content);
 			}
 			e.printStackTrace();
 		}
@@ -123,6 +434,7 @@
 	}
 	
 	public static String getUsername(Long id){
+
 		try {
 			for(VDCUser user : getAllUsers()){
 				if(user.getId().equals(id)){
@@ -169,6 +481,7 @@
 			e.printStackTrace();
 		}
 		
+		
 		return list;	
 	}
 	
@@ -202,32 +515,46 @@
 		return null;
 	}
 
-	public static boolean isUserInGroups(VDCUser user, List<String> userGroups) {
-		// find user who named userName belongs in what groups. check if the user is in any of the groups in userGroup list
-		try {
-			JSONArray groupArray = user.getUserGroups();
-
-			for(int i=0; i<groupArray.length(); i++){
-				JSONObject groupString = groupArray.getJSONObject(i); 
-				
-				String friendlyName = (String) groupString.get("friendlyName");
-				
-				for(String u : userGroups) {
-					if(StringUtils.equals(u, friendlyName)) {
-						return true;
+	public static boolean isUserInGroups(String userName, List<String> userGroups) throws IOException {
+		/* Get group members for each group in userGroup list, then check if user in any of them
+		 * get group members in a group by dataverse Api: GET http://$server/api/dataverses/$id/groups?key=$apiToken
+		 */
+		
+		String apiToken = PropertiesUtils.getPropValue("dvn_apitoken");
+		
+		for (String aGroup: userGroups) {
+			String query = PropertiesUtils.getPropValue("dvn_server") + "/api/dataverses/2/groups/"+ aGroup +"?key=" + apiToken;
+			//System.out.println("URL: " + query);
+		
+			HttpStringResponse response = null;
+			try {
+				response = HTTPUtils.getStringResponse(query);
+				JSONObject json = new JSONObject(response.content);
+			
+				if(StringUtils.equals(json.getString("status"), "OK")){
+					JSONObject jsonData = json.getJSONObject("data");
+					JSONArray jsonContainedRoleAssignees = jsonData.getJSONArray("containedRoleAssignees");
+					for (int i = 0; i < jsonContainedRoleAssignees.length(); i ++) {
+						String aName = jsonContainedRoleAssignees.getString(i);
+						if (aName.compareTo("@" + userName) == 0) {
+							System.out.println("user "+ aName +" in group: " + aGroup);
+							return true;	
+						}
 					}
+			
 				}
-				
+			} catch (Exception e) {
+				if(response != null){
+					System.err.println(response.content);
+				}
+				e.printStackTrace();
 			}
-							
-		} catch (JSONException e) {
-			
-			e.printStackTrace();
+		
 		}
 		
 		return false;
 	}
 
-	
-	
+
+
 }

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/mpiwg/gazetteer/dataverse/bo/Datasets.java	Fri Sep 11 10:34:25 2015 +0200
@@ -0,0 +1,117 @@
+package de.mpiwg.gazetteer.dataverse.bo;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class Datasets {
+
+	private Long id;
+	private String authority;
+	private String persistentURL;
+	private String protocol;
+	private String identifier;
+	private String createTime;
+	private String title;
+	private String status;
+	private String version;
+	private String versionState;
+
+	private String dvIdInDataverse;
+	
+	public Datasets(JSONObject json) throws JSONException{
+		
+		this.id = json.getLong("id");		
+		this.authority = json.getString("authority");
+		this.persistentURL = json.getString("persistentURL");
+		this.protocol = json.getString("protocol");
+		this.identifier = json.getString("identifier");
+		this.createTime = json.getString("createTime");
+		this.title = json.getString("title");
+		this.status = json.getString("status");
+		this.version = json.getString("version");
+		this.versionState = json.getString("versionState");
+		this.dvIdInDataverse = json.getString("dvIdInDataverse");
+		
+	}
+	
+	
+	public String getDvIdInDataverse() {
+		return dvIdInDataverse;
+	}
+
+
+	public void setDvIdInDataverse(String dvIdInDataverse) {
+		this.dvIdInDataverse = dvIdInDataverse;
+	}
+
+
+	public String getIdentifier() {
+		return identifier;
+	}
+
+	public void setIdentifier(String identifier) {
+		this.identifier = identifier;
+	}
+
+	public String getAuthority() {
+		return authority;
+	}
+	public void setAuthority(String authority) {
+		this.authority = authority;
+	}
+	
+	public String getPersistentURL() {
+		return persistentURL;
+	}
+	public void setPersistentURL(String persistentURL) {
+		this.persistentURL = persistentURL;
+	}
+	public String getProtocol() {
+		return protocol;
+	}
+	public void setProtocol(String protocol) {
+		this.protocol = protocol;
+	}
+	public String getVersion() {
+		return version;
+	}
+	public void setVersion(String version) {
+		this.version = version;
+	}
+	
+	public String getCreateTime() {
+		return createTime;
+	}
+	public void setCreateTime(String createTime) {
+		this.createTime = createTime;
+	}
+	public String getTitle() {
+		return title;
+	}
+	public void setTitle(String title) {
+		this.title = title;
+	}
+	public Long getId() {
+		return id;
+	}
+	public void setId(Long id) {
+		this.id = id;
+	}
+	public String getStatus() {
+		return status;
+	}
+	public void setStatus(String status) {
+		this.status = status;
+	}
+
+	public String getVersionState() {
+		return versionState;
+	}
+
+	public void setVersionState(String versionState) {
+		this.versionState = versionState;
+	}
+
+
+	
+}

--- a/src/main/java/de/mpiwg/gazetteer/dataverse/bo/VDCUser.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/dataverse/bo/VDCUser.java	Fri Sep 11 10:34:25 2015 +0200
@@ -43,14 +43,14 @@
     	this.id = JSONUtils.getLong(json, "id");
     	this.position = JSONUtils.getLong(json, "position");
     	this.version = JSONUtils.getLong(json, "version");
-    	
+    	/*
     	try {
 			this.userGroups = json.getJSONArray("userGroups");
 		} catch (JSONException e) {
-			// TODO Auto-generated catch block
 			this.userGroups = null;
 			e.printStackTrace();
 		}
+		*/
     }
 
 	public String getUserName() {

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/main/java/de/mpiwg/gazetteer/rest/GetTabDelimited4File.java	Fri Sep 11 10:34:25 2015 +0200
@@ -0,0 +1,55 @@
+package de.mpiwg.gazetteer.rest;
+
+import java.io.PrintWriter;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.lang.StringUtils;
+import org.json.JSONObject;
+
+import de.mpiwg.gazetteer.bo.LGFile;
+import de.mpiwg.gazetteer.utils.DataProvider;
+import de.mpiwg.gazetteer.utils.HTTPUtils;
+
+public class GetTabDelimited4File extends AbstractServletMethod {
+	public static String name = "getTabDelimited4File";
+	
+	public static void execute(HttpServletRequest request, HttpServletResponse response) throws Exception{
+		
+		Long fileId = getQueryLongParam(request, "fileId");
+		
+		if(fileId != null){
+			LGFile file = DataProvider.getInstance().getFile(fileId);
+			if(file != null){
+				String text = HTTPUtils.getTabDelimitedOfFile(fileId);	// get text as tab-delimited file
+					
+				if(StringUtils.isNotEmpty(text)){
+					
+					PrintWriter out = response.getWriter();
+					out.print(text);
+					out.flush();
+					response.setContentType("text/plain; charset=UTF-8");
+					
+				}	
+			} else{
+				response.setContentType("application/json");
+				JSONObject json = new JSONObject();
+				json.put("status", "error");
+				json.put("message", "File no found (" + fileId + ")");
+				PrintWriter out = response.getWriter();
+				out.print(json.toString());
+				out.flush();
+			}
+		}else{
+			response.setContentType("application/json");
+			JSONObject json = new JSONObject();
+			json.put("status", "error");
+			json.put("message", "Following parameters are mandatory: fileId.");
+			PrintWriter out = response.getWriter();
+			out.print(json.toString());
+			out.flush();
+		}
+	}
+	
+}

--- a/src/main/java/de/mpiwg/gazetteer/rest/GetTable4File.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/rest/GetTable4File.java	Fri Sep 11 10:34:25 2015 +0200
@@ -22,7 +22,8 @@
 		if(fileId != null){
 			LGFile file = DataProvider.getInstance().getFile(fileId);
 			if(file != null){
-				String text = HTTPUtils.getTableOfFile(fileId);
+				String text = HTTPUtils.getTableOfFile(fileId);	// get text as html table
+			
 				if(StringUtils.isNotEmpty(text)){
 					
 					PrintWriter out = response.getWriter();

--- a/src/main/java/de/mpiwg/gazetteer/rest/TextServlet.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/rest/TextServlet.java	Fri Sep 11 10:34:25 2015 +0200
@@ -63,6 +63,8 @@
 			GetSectionMetadata.execute(request, response);
 		}else if(StringUtils.equals(GetTable4File.name, method)){
 			GetTable4File.execute(request, response);
+		}else if(StringUtils.equals(GetTabDelimited4File.name, method)){
+			GetTabDelimited4File.execute(request, response);
 		}else{
 			writeError(response, "Content-type wrong. It should be: multipart/form-data");
 		}

--- a/src/main/java/de/mpiwg/gazetteer/utils/DBService.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/utils/DBService.java	Fri Sep 11 10:34:25 2015 +0200
@@ -464,7 +464,7 @@
 				
 			}
 			rs.close();
-		} catch (Exception e) {
+		} catch (Exception e) {	
 			e.printStackTrace();
 		} finally {
 			conn.close();
@@ -472,6 +472,43 @@
 		return section;
 	}
 	
+	public static DBSection getDeprecatedSectionWithContent(Long sectionId) throws SQLException {
+
+		Connection conn = null;
+		Statement stmt = null;
+		DBSection section = null;
+		
+		String query = "SELECT * FROM sections_revisions WHERE sections_id = '" + sectionId + "'";
+		
+		try {
+			Class.forName(JDBC_DRIVER);
+			conn = getNewConnection();
+			stmt = conn.createStatement();
+			
+			ResultSet rs = stmt.executeQuery(query);
+			rs.next();
+			//while (rs.next()) {
+				
+				section = new DBSection(rs);
+				
+				String content = getContent(conn, section);
+				section.setText(content);
+				
+				DBBook book = DBService.getInstance().getBook(section.getBookId());
+				section.setBook(book);
+				
+			//}
+			rs.close();
+		} catch (Exception e) {	
+			e.printStackTrace();
+		} finally {
+			conn.close();
+		}
+		return section;
+	}
+	
+	
+	
 	public static String fixToNewline(String orig){
 	    char[] chars = orig.toCharArray();
 	    StringBuilder sb = new StringBuilder(100);
@@ -494,8 +531,7 @@
 	//private static String getContent(Connection conn, String bookId, Integer startLine, Integer endLine) throws Exception{
 	private static String getContent(Connection conn, DBSection section) throws Exception{
 		String query = "SELECT content, line FROM contents WHERE books_id = '" + section.getBookId() + "' AND line >= '" + section.getStart_page() + "' AND line <= '" + section.getEnd_page() + "'";
-		
-		logger.debug(query);
+		//logger.debug(query);
 		
 		Statement stmt   = conn.createStatement();
 		ResultSet rs = stmt.executeQuery(query);
@@ -727,5 +763,6 @@
 		
 		
 	}
+
 	
 }

--- a/src/main/java/de/mpiwg/gazetteer/utils/HTTPUtils.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/gazetteer/utils/HTTPUtils.java	Fri Sep 11 10:34:25 2015 +0200
@@ -98,6 +98,59 @@
 		return sb.toString();
 	}
     
+    public static String getTabDelimitedOfFile(Long fileId) throws Exception{
+		
+		
+		String link = PropertiesUtils.getPropValue("extraction_interface") + "/Extractapp/ExportTabDelimited";
+		
+		System.out.println(link);
+		
+		URL url = new URL(link);
+		HttpURLConnection httpConn = (HttpURLConnection)url.openConnection();
+		
+		httpConn.setReadTimeout(10000);
+		httpConn.setConnectTimeout(15000);
+		httpConn.setRequestMethod("POST");
+		httpConn.setDoInput(true);
+		httpConn.setDoOutput(true);
+
+		//Send request
+		DataOutputStream wr = new DataOutputStream (httpConn.getOutputStream ());
+	    wr.writeBytes ("fileId=" + fileId);
+	    wr.flush ();
+	    wr.close ();
+		
+        BufferedReader in = null;
+        if (httpConn.getResponseCode() >= 400) {
+        	in = new BufferedReader(
+                    new InputStreamReader(
+                    		httpConn.getErrorStream()));
+        	
+        	String inputLine;
+            StringBuilder sb = new StringBuilder();
+            while ((inputLine = in.readLine()) != null) 
+            	sb.append(inputLine + "\n");
+        	
+        	throw new Exception("HTTP Error, code " + httpConn.getResponseCode());
+        	
+        } else {
+        	in = new BufferedReader(
+                    new InputStreamReader(
+                    		httpConn.getInputStream()));
+        }
+        
+        String inputLine;
+        StringBuilder sb = new StringBuilder();
+        while ((inputLine = in.readLine()) != null) 
+        	sb.append(inputLine + "\n");
+        in.close();
+        
+		return sb.toString();
+	}
+    
+    
+   
+    
     public static HttpStringResponse getStringResponse(String url) throws IOException, KeyManagementException, NoSuchAlgorithmException{
     	if(url.startsWith("https") || url.startsWith("HTTPS"))
     		return getHttpSSLStringResponse(url);

--- a/src/main/java/de/mpiwg/web/jsp/BranchPage.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/web/jsp/BranchPage.java	Fri Sep 11 10:34:25 2015 +0200
@@ -36,7 +36,8 @@
 	
 	private Long userId;
 
-	private String studyGlobalId;
+	
+	private String datasetId;
 	
 	
 	public void loadParameters(HttpServletRequest request, HttpServletResponse response){
@@ -44,10 +45,46 @@
 		this.response = response;
 		this.userId = getLongParameter("userId");
 		this.fileId = getLongParameter("fileId");
-		this.studyGlobalId = getParameter("studyGlobalId");
+		this.datasetId = getParameter("datasetId");
+		
 	}
 	
 	public void publishFile(){
+		System.out.println("publishFile: " + this.fileId + " in " + datasetId);
+		if(fileId != null && StringUtils.isNotEmpty(datasetId)){
+		
+			try {
+				JSONObject resp = DataverseUtils.publishFile(fileId, datasetId, getSessionBean().getUser().getUserName(), getSessionBean().getUser().getPassword());
+				System.out.println("resp: "+ resp);
+				
+				if(resp.has("status") && StringUtils.equals(resp.getString("status"), "ok")){
+					// update record for the uploaded file
+					LGFile file = DataProvider.getInstance().getFile(getFileId());
+					long dvId = Long.parseLong(this.getDatasetId());	// dvId means datasetId, and all dataset are in "Chinese Local Gazetteers" dataverse
+					file.setDvId(dvId);
+					
+					String datasetTitle = resp.getJSONObject("fileMetadata").getString("datasetTitle");
+					
+					file.setDatasetTitle(datasetTitle);
+					
+					DataProvider.getInstance().updateFile(file);
+					
+					addMsg("The File has been uploaded to dataverse.");
+					
+				} else if (StringUtils.equals(resp.getString("status"), "error")) {
+					addMsg("Dataverse reported an error: " + resp.getString("message"));
+				} else{
+					addMsg("Internal Error: problems generating the table for the file " + fileId);
+				}
+				
+			} catch (Exception e) {
+				internalError(e);
+			}
+		}
+	}
+	
+	/*
+	public void publishFile_removed(){
 		System.out.println("publishFile: " + this.fileId + " in " + studyGlobalId);
 		if(fileId != null && StringUtils.isNotEmpty(studyGlobalId)){
 			
@@ -84,6 +121,8 @@
 		}
 	}
 	
+	*/
+	
 	public void loadBranch(String branchId0){
 		try {
 			
@@ -255,11 +294,13 @@
 	public void setFileId(Long fileId) {
 		this.fileId = fileId;
 	}
-	public String getStudyGlobalId() {
-		return studyGlobalId;
+
+	public String getDatasetId() {
+		return datasetId;
 	}
 
-	public void setStudyGlobalId(String studyGlobalId) {
-		this.studyGlobalId = studyGlobalId;
+	public void setDatasetId(String datasetId) {
+		this.datasetId = datasetId;
 	}
+	
 }

--- a/src/main/java/de/mpiwg/web/jsp/HomePage.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/web/jsp/HomePage.java	Fri Sep 11 10:34:25 2015 +0200
@@ -11,6 +11,7 @@
 
 
 
+
 import de.mpiwg.gazetteer.bo.LGBranch;
 import de.mpiwg.gazetteer.utils.DataProvider;
 
@@ -40,9 +41,14 @@
 		this.branches = new ArrayList<LGBranch>();
 		if(getSessionBean().getUser() != null){
 			for(LGBranch branch : DataProvider.getInstance().getBranches(getSessionBean().getUser().getId())){
-	
 				branch.loadTransientData();
-				this.branches.add(branch);
+				// section of this branch may be deleted. In this case, we don't add the branch to branches list
+				// TODO might let user to delete the branch?
+				if (branch.isEmpty()) {
+					logger.debug("section of the branch doesn't exist anymore.");
+				} else {
+					this.branches.add(branch);
+				}
 			}	
 		}
 	}

--- a/src/main/java/de/mpiwg/web/jsp/SessionBean.java	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/java/de/mpiwg/web/jsp/SessionBean.java	Fri Sep 11 10:34:25 2015 +0200
@@ -83,9 +83,11 @@
 				theUserGroups.add("mpiwg");
 				theUserGroups.add("nuist");
 				
-				if (DataverseUtils.isUserInGroups(this.user, theUserGroups)){
-					logger.debug("user's groups: " + user.getUserGroups());
-
+				// test
+				theUserGroups.add("mygroup2");
+				
+				if (DataverseUtils.isUserInGroups(this.user.getUserName(), theUserGroups)){
+					
 					this.user.setPassword(this.password);
 					this.homePage.loadParameters(request, response);
 					this.homePage.reloadBranches();	
@@ -95,9 +97,10 @@
 					
 					addMsg("Please contact/email us to activate your account!");
 				}
+				
 				// -----
 			}else{
-				addMsg("User account no found or userName and password do not match!");
+				addMsg("User account not found, or username and password do not match!");
 			}
 						
 		} catch (Exception e) {

--- a/src/main/resources/config.properties	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/resources/config.properties	Fri Sep 11 10:34:25 2015 +0200
@@ -4,8 +4,10 @@
 files_root=/gazetteer-server/data
 
 #Local
-dvn_server=http://localhost/dvn
-#dvn_server=http://localhost:8082		
+#dvn_server=http://localhost/dvn
+dvn_server=http://localhost:8082
+dvn_apitoken=9dd1f749-8c42-49ab-a2ba-fbb963c2ff90
+dv_id=2
 root_server=http://localhost:8080/LGServices
 toc_interface=http://localhost:1080/localgazetteers-dev/LGToc
 extraction_interface=http://localhost:1080/localgazetteers-dev/extraction-interface

--- a/src/main/webapp/WEB-INF/web.xml	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/webapp/WEB-INF/web.xml	Fri Sep 11 10:34:25 2015 +0200
@@ -28,6 +28,7 @@
 			<param-name>requestEncoding</param-name>
 			<param-value>UTF-8</param-value>
 		</init-param>
+		
 	</filter>
 
 	<filter-mapping>

--- a/src/main/webapp/methods/getDataverseForm.jsp	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/webapp/methods/getDataverseForm.jsp	Fri Sep 11 10:34:25 2015 +0200
@@ -1,11 +1,14 @@
 <%@page import="de.mpiwg.gazetteer.dataverse.DataverseUtils"%>
 <%@page import="de.mpiwg.gazetteer.dataverse.bo.Study"%>
+<%@page import="de.mpiwg.gazetteer.dataverse.bo.Datasets"%>
 <%@page import="java.util.List"%>
 <jsp:useBean id="sessionBean" class="de.mpiwg.web.jsp.SessionBean" scope="session" />
 <%
 	
-
-	List<Study> studies = DataverseUtils.getStudies(sessionBean.getUser().getUserName(), sessionBean.getUser().getPassword());
+	//List<Study> studies = DataverseUtils.getStudies(sessionBean.getUser().getUserName(), sessionBean.getUser().getPassword());
+	// List<Datasets> studies = DataverseUtils.getDatasets(sessionBean.getUser().getUserName(), sessionBean.getUser().getPassword());
+	List<Datasets> studies = DataverseUtils.getDatasetsInAllDataverse(sessionBean.getUser().getUserName(), sessionBean.getUser().getPassword());
+	
 	System.out.println("%%%%% Studies: " + studies.size());
 	
 	if(studies.isEmpty()){
@@ -27,38 +30,40 @@
 	<table class="pageTable">
 		<tbody>
 			<tr>
-				<th><label class="tableTitle">Study Global ID</label></th>
-				<th><label class="tableTitle">Title</label></th>
-				<th><label class="tableTitle">Creator</label></th>
+				<th><label class="tableTitle">Dataverse</label></th>
+				<!-- 
+				<th><label class="tableTitle">Dataset ID</label></th>
+				-->
+				<th><label class="tableTitle">Dataset</label></th>
 				<th><label class="tableTitle">Created</label></th>
-				<th><label class="tableTitle">Status</label></th>
-				<th><label class="tableTitle">Dataverse</label></th>
+				<th><label class="tableTitle">State</label></th>
+				<th><label class="tableTitle">Version</label></th>
 				<th><label class="tableTitle">Publish</label></th>
 				<!-- 
 				<th><label class="tableTitle">Release Date</label></th>
-				<th><label class="tableTitle">Version</label></th>
 				<th><label class="tableTitle">Last Updated</label></th>
-				 -->
-				
-				<!-- 
 				<th><label class="tableTitle">Number of Files </label></th>
 				 -->
 			</tr>
 		</tbody>
 		<%
-			for(Study study : studies){
+			for(Datasets study : studies){
 		%>
 			
 			<tr>
-				<td><%= study.getGlobalId()%></td>
+				<!--  <td>Chinese Local Gazetteers </td> -->
+				<td><%= study.getDvIdInDataverse() %></td>
+				
+				<!-- <td><%= study.getId()%></td> 
+				-->
 				<td><%= study.getTitle() %></td>
-				<td><%= study.getCreator() %></td>
 				<td><%= study.getCreateTime() %></td>
-				<td><%= study.getStatus() %></td>
-				<td><%= study.getDataverse().getName() %></td>
+				<td><%= study.getVersionState().toLowerCase() %></td>
+				
+				<td><%= study.getVersion() %></td>
 				<td>
 					<input type="image" 
-						onclick="<%=sessionBean.getApplicationBean().getJSConfirmationPublish() %> setAction0('publishFile', 'dataverseForm', 'studyGlobalId', '<%=study.getGlobalId() %>');" 
+						onclick="<%=sessionBean.getApplicationBean().getJSConfirmationPublish() %> setAction0('publishFile', 'dataverseForm', 'datasetId', '<%=study.getId() %>');" 
 						src="<%=sessionBean.getApplicationBean().getPublishImage()%>"/>
 				</td>
 			</tr>

--- a/src/main/webapp/pages/branchPage.jsp	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/webapp/pages/branchPage.jsp	Fri Sep 11 10:34:25 2015 +0200
@@ -61,7 +61,7 @@
 <body>
 	<jsp:include page="../componentes/template.jsp"/>
 	
-	<div id="dialogDataverse" title="Select a study to publish your File" >
+	<div id="dialogDataverse" title="Select a Dataset to publish your File" >
 	  <div id="dialogDataverseTable">XXXX</div>
 	</div>		
 
@@ -271,7 +271,8 @@
 											<th><label class="tableTitle">User</label></th>
 											<th><label class="tableTitle">Created</label></th>
 											<th><label class="tableTitle">Text</label></th>
-											<th><label class="tableTitle">Table (to be published)</label></th>
+											<th><label class="tableTitle">View in table</label></th>
+											<th><label class="tableTitle">Text(to be published)</label></th>
 											<th><label class="tableTitle">Published in Dataverse?</label></th>
 											
 											<th><label class="tableTitle">Delete</label></th>
@@ -294,15 +295,23 @@
 														target="_blank">
 														<img alt="Show text" src="<%=sessionBean.getApplicationBean().getShowImage()%>"/>
 													</a>												
-												</td>												
+												</td>	
+												
+												<!-- TODO test: zoe add to test tab-separated file -->
+												<td>
+													<a href="<%=sessionBean.getApplicationBean().getRootServer() %>/rest/text/getTabDelimited4File?fileId=<%=file.getId() %>"
+														target="_blank">
+														<img alt="Show text" src="<%=sessionBean.getApplicationBean().getShowImage()%>"/>
+													</a>												
+												</td>	
+																						
+																						
+													
 												<td>
 													<% if(file.getDvId() == null) {%>
-													<button type="button" class="get-studies" data-file-id="<%=file.getId()%>">load</button>
+													<button type="button" class="get-studies" data-file-id="<%=file.getId()%>">Publish</button>
 													<% } else { %>
-													<label>Yes! Id in Dataverse: <%= file.getDvId() %></label>	
-												
-											 		<!-- TODO: replace "Id in Dataverse" as "file name in Dataverse" ? --
-												
+													<label>Yes! In dataset: <%= file.getDatasetTitle() %></label>	
 																			
 													<% } %>		
 												</td>												

--- a/src/main/webapp/pages/home.jsp	Tue Jun 30 14:31:29 2015 +0200
+++ b/src/main/webapp/pages/home.jsp	Fri Sep 11 10:34:25 2015 +0200
@@ -41,6 +41,7 @@
 						<td><label class="tableTitle">Branch ID</label></td>
 						<td><label class="tableTitle">Book ID</label></td>
 						<td><label class="tableTitle">Book Name</label></td>
+						<td><label class="tableTitle">Period</label></td>
 						<td><label class="tableTitle">Section Name</label></td>
 						<td><label class="tableTitle">Label</label></td>
 						<td><label class="tableTitle">Contributors</label></td>
@@ -52,12 +53,18 @@
 					</tr>	
 					
 					<% for(LGBranch branch : sessionBean.getHomePage().getBranches()) { %>
-					
 					<tr>
 						<td><%=branch.getId() %></td>
 						<td><%=branch.getBook().getId() %></td>
 						<td><%=branch.getBook().getName() %></td>
-						<td><%=branch.getSection().getName() %></td>
+						<td><%=branch.getBook().getPeriod() %></td>
+						
+						<td><%=branch.getSection().getName() %> 
+							<% if (branch.isDeprecated()) { %>
+							<label style="color:red">*** This is a deprecated section ***</label>
+							<% } %>
+						</td>
+						
 						<td><%=branch.getLabel() %></td>
 						<td>
 							<table style="width:120px;">