LGDataverses: src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java annotate

author	Zoe Hong <zhong@mpiwg-berlin.mpg.de>
date	Tue, 08 Sep 2015 17:00:21 +0200
parents
children

rev	line source
10 a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	1 /*
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	2 Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	3
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	4 Licensed under the Apache License, Version 2.0 (the "License");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	5 you may not use this file except in compliance with the License.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	6 You may obtain a copy of the License at
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	7
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	8 http://www.apache.org/licenses/LICENSE-2.0
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	9
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	10 Unless required by applicable law or agreed to in writing, software
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	11 distributed under the License is distributed on an "AS IS" BASIS,
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	13 See the License for the specific language governing permissions and
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	14 limitations under the License.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	15
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	16 Dataverse Network - A web application to share, preserve and analyze research data.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	17 Developed at the Institute for Quantitative Social Science, Harvard University.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	18 Version 3.0.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	19 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	20
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	21 package edu.harvard.iq.dataverse.dataaccess;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	22
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	23 import edu.harvard.iq.dataverse.DataFile;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	24 import edu.harvard.iq.dataverse.datavariable.DataVariable;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	25 import java.util.*;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	26 import java.util.Scanner;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	27 import java.util.logging.*;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	28 import java.io.*;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	29 import java.io.FileNotFoundException;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	30 import java.math.BigDecimal;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	31 import java.math.MathContext;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	32 import java.math.RoundingMode;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	33 import java.nio.ByteBuffer;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	34 import java.nio.channels.FileChannel;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	35 import java.nio.file.Paths;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	36 import java.nio.file.StandardOpenOption;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	37 import java.util.regex.Matcher;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	38
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	39
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	40 import org.apache.commons.lang.*;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	41
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	42
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	43 /**
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	44 *
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	45 * @author Leonid Andreev
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	46 * original author:
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	47 * @author a.sone
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	48 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	49
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	50 public class TabularSubsetGenerator implements SubsetGenerator {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	51
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	52 private static Logger dbgLog = Logger.getLogger(TabularSubsetGenerator.class.getPackage().getName());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	53
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	54 private static int COLUMN_TYPE_STRING = 1;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	55 private static int COLUMN_TYPE_LONG = 2;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	56 private static int COLUMN_TYPE_DOUBLE = 3;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	57 private static int COLUMN_TYPE_FLOAT = 4;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	58
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	59 private static int MAX_COLUMN_BUFFER = 8192;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	60
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	61 private FileChannel fileChannel = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	62
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	63 private int varcount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	64 private int casecount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	65 private int subsetcount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	66
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	67 private byte[][] columnEntries = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	68
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	69
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	70 private ByteBuffer[] columnByteBuffers;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	71 private int[] columnBufferSizes;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	72 private int[] columnBufferOffsets;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	73
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	74 private long[] columnStartOffsets;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	75 private long[] columnTotalOffsets;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	76 private long[] columnTotalLengths;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	77
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	78 public TabularSubsetGenerator() {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	79
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	80 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	81
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	82 public TabularSubsetGenerator (DataFile datafile, List<DataVariable> variables) throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	83 if (!datafile.isTabularData()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	84 throw new IOException("DataFile is not tabular data.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	85 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	86
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	87 setVarCount(datafile.getDataTable().getVarQuantity().intValue());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	88 setCaseCount(datafile.getDataTable().getCaseQuantity().intValue());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	89
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	90 File tabfile = datafile.getFileSystemLocation().toFile();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	91 File rotatedImageFile = getRotatedImage(tabfile, getVarCount(), getCaseCount());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	92 long[] columnEndOffsets = extractColumnOffsets(rotatedImageFile, getVarCount(), getCaseCount());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	93
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	94 fileChannel = (FileChannel.open(Paths.get(rotatedImageFile.getAbsolutePath()), StandardOpenOption.READ));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	95
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	96 if (variables == null \|\| variables.size() < 1 \|\| variables.size() > getVarCount()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	97 throw new IOException("Illegal number of variables in the subset request");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	98 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	99
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	100 subsetcount = variables.size();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	101 columnTotalOffsets = new long[subsetcount];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	102 columnTotalLengths = new long[subsetcount];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	103 columnByteBuffers = new ByteBuffer[subsetcount];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	104
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	105
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	106
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	107 if (subsetcount == 1) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	108 if (!datafile.getDataTable().getId().equals(variables.get(0).getDataTable().getId())) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	109 throw new IOException("Variable in the subset request does not belong to the datafile.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	110 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	111 dbgLog.fine("single variable subset; setting fileChannel position to "+extractColumnOffset(columnEndOffsets, variables.get(0).getFileOrder()));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	112 fileChannel.position(extractColumnOffset(columnEndOffsets, variables.get(0).getFileOrder()));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	113 columnTotalLengths[0] = extractColumnLength(columnEndOffsets, variables.get(0).getFileOrder());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	114 columnTotalOffsets[0] = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	115 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	116 columnEntries = new byte[subsetcount][];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	117
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	118 columnBufferSizes = new int[subsetcount];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	119 columnBufferOffsets = new int[subsetcount];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	120 columnStartOffsets = new long[subsetcount];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	121
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	122 int i = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	123 for (DataVariable var : variables) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	124 if (!datafile.getDataTable().getId().equals(var.getDataTable().getId())) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	125 throw new IOException("Variable in the subset request does not belong to the datafile.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	126 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	127 columnByteBuffers[i] = ByteBuffer.allocate(MAX_COLUMN_BUFFER);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	128 columnTotalLengths[i] = extractColumnLength(columnEndOffsets, var.getFileOrder());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	129 columnStartOffsets[i] = extractColumnOffset(columnEndOffsets, var.getFileOrder());
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	130 if (columnTotalLengths[i] < MAX_COLUMN_BUFFER) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	131 columnByteBuffers[i].limit((int)columnTotalLengths[i]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	132 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	133 fileChannel.position(columnStartOffsets[i]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	134 columnBufferSizes[i] = fileChannel.read(columnByteBuffers[i]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	135 columnBufferOffsets[i] = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	136 columnTotalOffsets[i] = columnBufferSizes[i];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	137 i++;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	138 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	139 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	140 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	141
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	142 private int getVarCount() {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	143 return varcount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	144 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	145
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	146 private void setVarCount(int varcount) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	147 this.varcount = varcount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	148 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	149
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	150 private int getCaseCount() {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	151 return casecount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	152 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	153
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	154 private void setCaseCount(int casecount) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	155 this.casecount = casecount;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	156 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	157
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	158
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	159 /*
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	160 * Note that this method operates on the absolute column number, i.e.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	161 * the number of the physical column in the tabular file. This is stored
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	162 * in DataVariable.FileOrder.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	163 * This "column number" should not be confused with the number of column
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	164 * in the subset request; a user can request any number of variable
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	165 * columns, in an order that doesn't have to follow the physical order
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	166 * of the columns in the file.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	167 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	168 private long extractColumnOffset(long[] columnEndOffsets, int column) throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	169 if (columnEndOffsets == null \|\| columnEndOffsets.length <= column) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	170 throw new IOException("Offsets table not initialized; or column out of bounds.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	171 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	172 long columnOffset;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	173
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	174 if (column > 0) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	175 columnOffset = columnEndOffsets[column - 1];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	176 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	177 columnOffset = getVarCount() * 8;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	178 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	179 return columnOffset;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	180 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	181
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	182 /*
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	183 * See the comment for the method above.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	184 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	185 private long extractColumnLength(long[] columnEndOffsets, int column) throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	186 if (columnEndOffsets == null \|\| columnEndOffsets.length <= column) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	187 throw new IOException("Offsets table not initialized; or column out of bounds.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	188 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	189 long columnLength;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	190
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	191 if (column > 0) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	192 columnLength = columnEndOffsets[column] - columnEndOffsets[column - 1];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	193 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	194 columnLength = columnEndOffsets[0] - varcount * 8;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	195 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	196
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	197 return columnLength;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	198 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	199
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	200
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	201 private void bufferMoreColumnBytes(int column) throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	202 if (columnTotalOffsets[column] >= columnTotalLengths[column]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	203 throw new IOException("attempt to buffer bytes past the column boundary");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	204 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	205 fileChannel.position(columnStartOffsets[column] + columnTotalOffsets[column]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	206
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	207 columnByteBuffers[column].clear();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	208 if (columnTotalLengths[column] < columnTotalOffsets[column] + MAX_COLUMN_BUFFER) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	209 dbgLog.fine("Limiting the buffer to "+(columnTotalLengths[column] - columnTotalOffsets[column])+" bytes");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	210 columnByteBuffers[column].limit((int) (columnTotalLengths[column] - columnTotalOffsets[column]));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	211 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	212 columnBufferSizes[column] = fileChannel.read(columnByteBuffers[column]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	213 dbgLog.fine("Read "+columnBufferSizes[column]+" bytes for subset column "+column);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	214 columnBufferOffsets[column] = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	215 columnTotalOffsets[column] += columnBufferSizes[column];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	216 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	217
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	218 /*
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	219 do not use this method!
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	220 there's a high potential for the "UTF8 character split between buffers" error!
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	221 public String readColumnEntry(int column) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	222 String ret = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	223 int currentbyte;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	224
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	225 if (columnBufferOffsets[column] >= columnBufferSizes[column]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	226 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	227 bufferMoreColumnBytes(column);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	228 } catch (IOException ioe) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	229 return null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	230 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	231 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	232
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	233 currentbyte = columnBufferOffsets[column];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	234 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	235 while (columnByteBuffers[column].array()[currentbyte] != '\n') {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	236 currentbyte++;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	237 if (currentbyte == columnBufferSizes[column]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	238 // save the leftover:
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	239 if (ret == null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	240 ret = new String(columnByteBuffers[column].array(), columnBufferOffsets[column], columnBufferSizes[column] - columnBufferOffsets[column], "UTF8");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	241 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	242 ret = ret.concat(new String(columnByteBuffers[column].array(), columnBufferOffsets[column], columnBufferSizes[column] - columnBufferOffsets[column], "UTF8"));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	243 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	244 // read more bytes:
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	245 bufferMoreColumnBytes(column);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	246 currentbyte = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	247 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	248 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	249
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	250 // presumably, we have found our '\n':
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	251 if (ret == null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	252 ret = new String(columnByteBuffers[column].array(), columnBufferOffsets[column], currentbyte - columnBufferOffsets[column], "UTF8");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	253 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	254 ret = ret.concat(new String(columnByteBuffers[column].array(), columnBufferOffsets[column], currentbyte - columnBufferOffsets[column], "UTF8"));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	255 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	256
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	257 } catch (IOException ioe) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	258 return null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	259 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	260
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	261 columnBufferOffsets[column] += (currentbyte + 1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	262
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	263 return ret;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	264 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	265 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	266
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	267 public byte[] readColumnEntryBytes(int column) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	268 return readColumnEntryBytes(column, true);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	269 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	270
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	271
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	272 public byte[] readColumnEntryBytes(int column, boolean addTabs) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	273 byte[] leftover = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	274 byte[] ret = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	275
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	276 if (columnBufferOffsets[column] >= columnBufferSizes[column]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	277 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	278 bufferMoreColumnBytes(column);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	279 if (columnBufferSizes[column] < 1) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	280 return null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	281 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	282 } catch (IOException ioe) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	283 return null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	284 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	285 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	286
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	287 int byteindex = columnBufferOffsets[column];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	288 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	289 while (columnByteBuffers[column].array()[byteindex] != '\n') {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	290 byteindex++;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	291 if (byteindex == columnBufferSizes[column]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	292 // save the leftover:
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	293 if (leftover == null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	294 leftover = new byte[columnBufferSizes[column] - columnBufferOffsets[column]];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	295 System.arraycopy(columnByteBuffers[column].array(), columnBufferOffsets[column], leftover, 0, columnBufferSizes[column] - columnBufferOffsets[column]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	296 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	297 byte[] merged = new byte[leftover.length + columnBufferSizes[column]];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	298
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	299 System.arraycopy(leftover, 0, merged, 0, leftover.length);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	300 System.arraycopy(columnByteBuffers[column].array(), 0, merged, leftover.length, columnBufferSizes[column]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	301 leftover = merged;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	302 merged = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	303 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	304 // read more bytes:
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	305 bufferMoreColumnBytes(column);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	306 if (columnBufferSizes[column] < 1) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	307 return null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	308 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	309 byteindex = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	310 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	311 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	312
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	313 // presumably, we have found our '\n':
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	314 if (leftover == null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	315 ret = new byte[byteindex - columnBufferOffsets[column] + 1];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	316 System.arraycopy(columnByteBuffers[column].array(), columnBufferOffsets[column], ret, 0, byteindex - columnBufferOffsets[column] + 1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	317 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	318 ret = new byte[leftover.length + byteindex + 1];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	319 System.arraycopy(leftover, 0, ret, 0, leftover.length);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	320 System.arraycopy(columnByteBuffers[column].array(), 0, ret, leftover.length, byteindex + 1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	321 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	322
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	323 } catch (IOException ioe) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	324 return null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	325 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	326
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	327 columnBufferOffsets[column] = (byteindex + 1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	328
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	329 if (column < columnBufferOffsets.length - 1) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	330 ret[ret.length - 1] = '\t';
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	331 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	332 return ret;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	333 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	334
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	335 public int readSingleColumnSubset(byte[] buffer) throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	336 if (columnTotalOffsets[0] == columnTotalLengths[0]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	337 return -1;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	338 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	339
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	340 if (columnByteBuffers[0] == null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	341 dbgLog.fine("allocating single column subset buffer.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	342 columnByteBuffers[0] = ByteBuffer.allocate(buffer.length);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	343 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	344
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	345 int bytesread = fileChannel.read(columnByteBuffers[0]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	346 dbgLog.fine("single column subset: read "+bytesread+" bytes.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	347 if (columnTotalOffsets[0] + bytesread > columnTotalLengths[0]) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	348 bytesread = (int)(columnTotalLengths[0] - columnTotalOffsets[0]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	349 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	350 System.arraycopy(columnByteBuffers[0].array(), 0, buffer, 0, bytesread);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	351
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	352 columnTotalOffsets[0] += bytesread;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	353 columnByteBuffers[0].clear();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	354 return bytesread > 0 ? bytesread : -1;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	355 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	356
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	357
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	358 public byte[] readSubsetLineBytes() throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	359 byte[] ret = null;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	360 int total = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	361
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	362 for (int i = 0; i < subsetcount; i++) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	363 columnEntries[i] = readColumnEntryBytes(i);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	364 if (columnEntries[i] == null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	365 throw new IOException("Failed to read subset line entry");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	366 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	367 total += columnEntries[i].length;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	368 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	369
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	370 ret = new byte[total];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	371 int offset = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	372 for (int i = 0; i < subsetcount; i++) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	373 System.arraycopy(columnEntries[i], 0, ret, offset, columnEntries[i].length);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	374 offset += columnEntries[i].length;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	375 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	376 dbgLog.fine("line: "+new String(ret));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	377 return ret;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	378 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	379
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	380
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	381 public void close() {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	382 if (fileChannel != null) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	383 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	384 fileChannel.close();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	385 } catch (IOException ioe) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	386 // don't care.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	387 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	388 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	389 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	390
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	391 public void subsetFile(String infile, String outfile, Set<Integer> columns, Long numCases) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	392 subsetFile(infile, outfile, columns, numCases, "\t");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	393 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	394
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	395 public void subsetFile(String infile, String outfile, Set<Integer> columns, Long numCases,
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	396 String delimiter) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	397 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	398 subsetFile(new FileInputStream(new File(infile)), outfile, columns, numCases, delimiter);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	399 } catch (IOException ex) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	400 throw new RuntimeException("Could not open file "+infile);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	401 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	402 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	403
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	404
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	405 public void subsetFile(InputStream in, String outfile, Set<Integer> columns, Long numCases,
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	406 String delimiter) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	407 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	408 Scanner scanner = new Scanner(in);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	409 scanner.useDelimiter("\\n");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	410
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	411 BufferedWriter out = new BufferedWriter(new FileWriter(outfile));
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	412 for (long caseIndex = 0; caseIndex < numCases; caseIndex++) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	413 if (scanner.hasNext()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	414 String[] line = (scanner.next()).split(delimiter,-1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	415 List<String> ln = new ArrayList<String>();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	416 for (Integer i : columns) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	417 ln.add(line[i]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	418 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	419 out.write(StringUtils.join(ln,"\t")+"\n");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	420 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	421 throw new RuntimeException("Tab file has fewer rows than the determined number of cases.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	422 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	423 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	424
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	425 while (scanner.hasNext()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	426 if (!"".equals(scanner.next()) ) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	427 throw new RuntimeException("Tab file has extra nonempty rows than the determined number of cases.");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	428
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	429 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	430 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	431
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	432 scanner.close();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	433 out.close();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	434
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	435 } catch (FileNotFoundException e) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	436 e.printStackTrace();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	437 } catch (IOException e) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	438 e.printStackTrace();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	439 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	440
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	441 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	442
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	443 /*
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	444 * Straightforward method for subsetting a column; inefficient on large
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	445 * files, OK to use on small files:
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	446 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	447
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	448 public static Double[] subsetDoubleVector(InputStream in, int column, int numCases) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	449 Double[] retVector = new Double[numCases];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	450 Scanner scanner = new Scanner(in);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	451 scanner.useDelimiter("\\n");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	452
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	453 for (int caseIndex = 0; caseIndex < numCases; caseIndex++) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	454 if (scanner.hasNext()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	455 String[] line = (scanner.next()).split("\t", -1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	456 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	457 retVector[caseIndex] = new Double(line[column]);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	458 } catch (NumberFormatException ex) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	459 retVector[caseIndex] = null; // missing value
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	460 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	461 } else {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	462 scanner.close();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	463 throw new RuntimeException("Tab file has fewer rows than the stored number of cases!");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	464 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	465 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	466
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	467 int tailIndex = numCases;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	468 while (scanner.hasNext()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	469 String nextLine = scanner.next();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	470 if (!"".equals(nextLine)) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	471 scanner.close();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	472 throw new RuntimeException("Column "+column+": tab file has more nonempty rows than the stored number of cases ("+numCases+")! current index: "+tailIndex+", line: "+nextLine);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	473 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	474 tailIndex++;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	475 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	476
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	477 scanner.close();
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	478 return retVector;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	479
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	480 }
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	481
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	482 /*
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	483 * Straightforward method for subsetting a tab-delimited data file, extracting
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	484 * all the columns representing continuous variables and returning them as
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	485 * a 2-dimensional array of Doubles;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	486 * Inefficient on large files, OK to use on small ones.
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	487 */
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	488 public static Double[][] subsetDoubleVectors(InputStream in, Set<Integer> columns, int numCases) throws IOException {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	489 Double[][] retVector = new Double[columns.size()][numCases];
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	490 Scanner scanner = new Scanner(in);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	491 scanner.useDelimiter("\\n");
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	492
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	493 for (int caseIndex = 0; caseIndex < numCases; caseIndex++) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	494 if (scanner.hasNext()) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	495 String[] line = (scanner.next()).split("\t", -1);
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	496 int j = 0;
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	497 for (Integer i : columns) {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	498 try {
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	499 // TODO: verify that NaN and +-Inf are going to be
a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0 Zoe Hong <zhong@mpiwg-berlin.mpg.de> parents: diff changeset	500 // handled correctly here! -- L.A.

10

a50cf11e5178 Rewrite LGDataverse completely upgrading to dataverse4.0