Example usage for org.apache.commons.csv CSVRecord size

Introduction

In this page you can find the example usage for org.apache.commons.csv CSVRecord size.

Prototype

public int size() 

Source Link

Document

Returns the number of values in this record.

Usage

From source file:edu.si.sidora.tabularmetadata.TabularMetadataGenerator.java

/**
 * The main entry point to application workflow.
 * //  w  w w. java  2  s .c  om
 * @param dataUrl Where to find some tabular data.
 * @param withHeaders whether this tabular data has a header row
 * @return The results of metadata extraction.
 * @throws IOException
 */
public TabularMetadata getMetadata(final URL dataUrl, final Boolean withHeaders) throws IOException {
    try (final CSVParser csvParser = parse(dataUrl, CHARACTER_ENCODING, format)) {
        final PeekingIterator<CSVRecord> parser = peekingIterator(csvParser.iterator());
        // TODO allow a HeaderHeuristic to use more information than the
        // first line of data
        final CSVRecord firstLine = parser.peek();
        final boolean hasHeaders;
        if (withHeaders == null) {
            log.debug("Checking for the existence of headers.");
            for (final String field : firstLine) {
                headerStrategy.accept(field);
            }
            hasHeaders = headerStrategy.results();
            headerStrategy.reset();
        } else {
            hasHeaders = withHeaders;
            log.debug("Accepted information that headers is {}.", hasHeaders);
        }
        final List<String> headerNames;
        if (hasHeaders) {
            headerNames = newArrayList(firstLine);
            log.debug("Found headers: {}", headerNames);
            if (parser.hasNext())
                parser.next();
        } else {
            headerNames = emptyHeaders(firstLine.size());
            log.debug("Found no headers.");
        }
        // scan values up to the limit
        final TabularScanner scanner = new TabularScanner(parser, typeStrategy, rangeStrategy, enumStrategy);
        scanner.scan(scanLimit);

        final List<TypeDeterminingHeuristic<?>> typeStrategies = scanner.getTypeStrategies();
        final List<RangeDeterminingHeuristic<?>> rangeStrategies = scanner.getRangeStrategies();
        final List<EnumeratedValuesHeuristic<?>> enumStrategies = scanner.getEnumStrategies();

        // extract the results for each field
        final List<DataType> columnTypes = typeStrategies.stream().map(Heuristic::results).collect(toList());
        final List<Map<DataType, Range<?>>> minMaxes = rangeStrategies.stream().map(Heuristic::results)
                .collect(toList());
        final List<Map<DataType, Set<String>>> enumValues = enumStrategies.stream().map(Heuristic::results)
                .collect(toList());
        final List<Ratio> valuesSeen = typeStrategies.stream()
                .map(h -> new Ratio(h.valuesSeen() - h.parseableValuesSeen(), h.valuesSeen()))
                .collect(toList());
        return new TabularMetadata(headerNames, valuesSeen, columnTypes, minMaxes, enumValues);
    } catch (final NoSuchElementException e) {
        throw new EmptyDataFileException(dataUrl + " has no data in it!");
    }
}

---

From source file:io.swagger.inflector.controllers.SwaggerOperationController.java

@Override
public Response apply(ContainerRequestContext ctx) {
    List<Parameter> parameters = operation.getParameters();
    final RequestContext requestContext = createContext(ctx);

    String path = ctx.getUriInfo().getPath();
    Map<String, Map<String, String>> formMap = new HashMap<String, Map<String, String>>();
    Map<String, File> inputStreams = new HashMap<String, File>();

    Object[] args = new Object[parameters.size() + 1];
    if (parameters != null) {
        int i = 0;

        args[i] = requestContext;/*from  w ww.j  a  va 2 s  . c om*/
        i += 1;
        List<ValidationMessage> missingParams = new ArrayList<ValidationMessage>();
        UriInfo uri = ctx.getUriInfo();
        String formDataString = null;
        String[] parts = null;
        Set<String> existingKeys = new HashSet<String>();

        for (Iterator<String> x = uri.getQueryParameters().keySet().iterator(); x.hasNext();) {
            existingKeys.add(x.next() + ": qp");
        }
        for (Iterator<String> x = uri.getPathParameters().keySet().iterator(); x.hasNext();) {
            existingKeys.add(x.next() + ": pp");
        }
        for (Iterator<String> x = ctx.getHeaders().keySet().iterator(); x.hasNext();) {
            String key = x.next();
            //              if(!commonHeaders.contains(key))
            //                existingKeys.add(key);
        }
        MediaType mt = requestContext.getMediaType();

        for (Parameter p : parameters) {
            Map<String, String> headers = new HashMap<String, String>();
            String name = null;

            if (p instanceof FormParameter) {
                if (formDataString == null) {
                    // can only read stream once
                    if (mt.isCompatible(MediaType.MULTIPART_FORM_DATA_TYPE)) {
                        // get the boundary
                        String boundary = mt.getParameters().get("boundary");

                        if (boundary != null) {
                            try {
                                InputStream output = ctx.getEntityStream();

                                MultipartStream multipartStream = new MultipartStream(output,
                                        boundary.getBytes());
                                boolean nextPart = multipartStream.skipPreamble();
                                while (nextPart) {
                                    String header = multipartStream.readHeaders();
                                    // process headers
                                    if (header != null) {
                                        CSVFormat format = CSVFormat.DEFAULT.withDelimiter(';')
                                                .withRecordSeparator("=");

                                        Iterable<CSVRecord> records = format.parse(new StringReader(header));
                                        for (CSVRecord r : records) {
                                            for (int j = 0; j < r.size(); j++) {
                                                String string = r.get(j);

                                                Iterable<CSVRecord> outerString = CSVFormat.DEFAULT
                                                        .withDelimiter('=').parse(new StringReader(string));
                                                for (CSVRecord outerKvPair : outerString) {
                                                    if (outerKvPair.size() == 2) {
                                                        String key = outerKvPair.get(0).trim();
                                                        String value = outerKvPair.get(1).trim();
                                                        if ("name".equals(key)) {
                                                            name = value;
                                                        }
                                                        headers.put(key, value);
                                                    } else {
                                                        Iterable<CSVRecord> innerString = CSVFormat.DEFAULT
                                                                .withDelimiter(':')
                                                                .parse(new StringReader(string));
                                                        for (CSVRecord innerKVPair : innerString) {
                                                            if (innerKVPair.size() == 2) {
                                                                String key = innerKVPair.get(0).trim();
                                                                String value = innerKVPair.get(1).trim();
                                                                if ("name".equals(key)) {
                                                                    name = value;
                                                                }
                                                                headers.put(key, value);
                                                            }
                                                        }
                                                    }
                                                }
                                                if (name != null) {
                                                    formMap.put(name, headers);
                                                }
                                            }
                                        }
                                    }
                                    String filename = extractFilenameFromHeaders(headers);
                                    if (filename != null) {
                                        try {
                                            File file = new File(Files.createTempDir(), filename);
                                            file.deleteOnExit();
                                            file.getParentFile().deleteOnExit();
                                            FileOutputStream fo = new FileOutputStream(file);
                                            multipartStream.readBodyData(fo);
                                            inputStreams.put(name, file);
                                        } catch (Exception e) {
                                            LOGGER.error("Failed to extract uploaded file", e);
                                        }
                                    } else {
                                        ByteArrayOutputStream bo = new ByteArrayOutputStream();
                                        multipartStream.readBodyData(bo);
                                        String value = bo.toString();
                                        headers.put(name, value);
                                    }
                                    if (name != null) {
                                        formMap.put(name, headers);
                                    }
                                    headers = new HashMap<>();
                                    name = null;
                                    nextPart = multipartStream.readBoundary();
                                }
                            } catch (IOException e) {
                                e.printStackTrace();
                            }
                        }
                    } else {
                        try {
                            formDataString = IOUtils.toString(ctx.getEntityStream(), "UTF-8");
                            parts = formDataString.split("&");

                            for (String part : parts) {
                                String[] kv = part.split("=");
                                existingKeys.add(kv[0] + ": fp");
                            }
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        }
        for (Parameter parameter : parameters) {
            String in = parameter.getIn();
            Object o = null;

            try {
                if ("formData".equals(in)) {
                    SerializableParameter sp = (SerializableParameter) parameter;
                    String name = parameter.getName();
                    if (mt.isCompatible(MediaType.MULTIPART_FORM_DATA_TYPE)) {
                        // look in the form map
                        Map<String, String> headers = formMap.get(name);
                        if (headers != null && headers.size() > 0) {
                            if ("file".equals(sp.getType())) {
                                o = inputStreams.get(name);
                            } else {
                                Object obj = headers.get(parameter.getName());
                                if (obj != null) {
                                    JavaType jt = parameterClasses[i];
                                    Class<?> cls = jt.getRawClass();

                                    List<String> os = Arrays.asList(obj.toString());
                                    try {
                                        o = validator.convertAndValidate(os, parameter, cls, definitions);
                                    } catch (ConversionException e) {
                                        missingParams.add(e.getError());
                                    } catch (ValidationException e) {
                                        missingParams.add(e.getValidationMessage());
                                    }
                                }
                            }
                        }
                    } else {
                        if (formDataString != null) {
                            for (String part : parts) {
                                String[] kv = part.split("=");
                                if (kv != null) {
                                    if (kv.length > 0) {
                                        existingKeys.remove(kv[0] + ": fp");
                                    }
                                    if (kv.length == 2) {
                                        // TODO how to handle arrays here?
                                        String key = kv[0];
                                        try {
                                            String value = URLDecoder.decode(kv[1], "utf-8");
                                            if (parameter.getName().equals(key)) {
                                                JavaType jt = parameterClasses[i];
                                                Class<?> cls = jt.getRawClass();
                                                try {
                                                    o = validator.convertAndValidate(Arrays.asList(value),
                                                            parameter, cls, definitions);
                                                } catch (ConversionException e) {
                                                    missingParams.add(e.getError());
                                                } catch (ValidationException e) {
                                                    missingParams.add(e.getValidationMessage());
                                                }
                                            }
                                        } catch (UnsupportedEncodingException e) {
                                            LOGGER.error("unable to decode value for " + key);
                                        }
                                    }
                                }
                            }
                        }
                    }
                } else {
                    try {
                        String paramName = parameter.getName();
                        if ("query".equals(in)) {
                            existingKeys.remove(paramName + ": qp");
                        }
                        if ("path".equals(in)) {
                            existingKeys.remove(paramName + ": pp");
                        }
                        JavaType jt = parameterClasses[i];
                        Class<?> cls = jt.getRawClass();
                        if ("body".equals(in)) {
                            if (ctx.hasEntity()) {
                                BodyParameter body = (BodyParameter) parameter;
                                o = EntityProcessorFactory.readValue(ctx.getMediaType(), ctx.getEntityStream(),
                                        cls);
                                if (o != null) {
                                    validate(o, body.getSchema(), SchemaValidator.Direction.INPUT);
                                }
                            } else if (parameter.getRequired()) {
                                ValidationException e = new ValidationException();
                                e.message(new ValidationMessage()
                                        .message("The input body `" + paramName + "` is required"));
                                throw e;
                            }
                        }
                        if ("query".equals(in)) {
                            o = validator.convertAndValidate(uri.getQueryParameters().get(parameter.getName()),
                                    parameter, cls, definitions);
                        } else if ("path".equals(in)) {
                            o = validator.convertAndValidate(uri.getPathParameters().get(parameter.getName()),
                                    parameter, cls, definitions);
                        } else if ("header".equals(in)) {
                            o = validator.convertAndValidate(ctx.getHeaders().get(parameter.getName()),
                                    parameter, cls, definitions);
                        }
                    } catch (ConversionException e) {
                        missingParams.add(e.getError());
                    } catch (ValidationException e) {
                        missingParams.add(e.getValidationMessage());
                    }
                }
            } catch (NumberFormatException e) {
                LOGGER.error("Couldn't find " + parameter.getName() + " (" + in + ") to " + parameterClasses[i],
                        e);
            }

            args[i] = o;
            i += 1;
        }
        if (existingKeys.size() > 0) {
            LOGGER.debug("unexpected keys: " + existingKeys);
        }
        if (missingParams.size() > 0) {
            StringBuilder builder = new StringBuilder();
            builder.append("Input error");
            if (missingParams.size() > 1) {
                builder.append("s");
            }
            builder.append(": ");
            int count = 0;
            for (ValidationMessage message : missingParams) {
                if (count > 0) {
                    builder.append(", ");
                }
                if (message != null && message.getMessage() != null) {
                    builder.append(message.getMessage());
                } else {
                    builder.append("no additional input");
                }
                count += 1;
            }
            int statusCode = config.getInvalidRequestStatusCode();
            ApiError error = new ApiError().code(statusCode).message(builder.toString());
            throw new ApiException(error);
        }
    }
    try {
        if (method != null) {
            LOGGER.info("calling method " + method + " on controller " + this.controller + " with args "
                    + Arrays.toString(args));
            try {
                Object response = method.invoke(controller, args);
                if (response instanceof ResponseContext) {
                    ResponseContext wrapper = (ResponseContext) response;
                    ResponseBuilder builder = Response.status(wrapper.getStatus());

                    // response headers
                    for (String key : wrapper.getHeaders().keySet()) {
                        List<String> v = wrapper.getHeaders().get(key);
                        if (v.size() == 1) {
                            builder.header(key, v.get(0));
                        } else {
                            builder.header(key, v);
                        }
                    }

                    // entity
                    if (wrapper.getEntity() != null) {
                        builder.entity(wrapper.getEntity());
                        // content type
                        if (wrapper.getContentType() != null) {
                            builder.type(wrapper.getContentType());
                        } else {
                            final ContextResolver<ContentTypeSelector> selector = providersProvider.get()
                                    .getContextResolver(ContentTypeSelector.class, MediaType.WILDCARD_TYPE);
                            if (selector != null) {
                                selector.getContext(getClass()).apply(ctx.getAcceptableMediaTypes(), builder);
                            }
                        }

                        if (operation.getResponses() != null) {
                            String responseCode = String.valueOf(wrapper.getStatus());
                            io.swagger.models.Response responseSchema = operation.getResponses()
                                    .get(responseCode);
                            if (responseSchema == null) {
                                // try default response schema
                                responseSchema = operation.getResponses().get("default");
                            }
                            if (responseSchema != null && responseSchema.getSchema() != null) {
                                validate(wrapper.getEntity(), responseSchema.getSchema(),
                                        SchemaValidator.Direction.OUTPUT);
                            } else {
                                LOGGER.debug(
                                        "no response schema for code " + responseCode + " to validate against");
                            }
                        }
                    }

                    return builder.build();
                }
                return Response.ok().entity(response).build();
            } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
                for (Throwable cause = e.getCause(); cause != null;) {
                    if (cause instanceof ApiException) {
                        throw (ApiException) cause;
                    }
                    final Throwable next = cause.getCause();
                    cause = next == cause || next == null ? null : next;
                }
                throw new ApiException(ApiErrorUtils.createInternalError(), e);
            }
        }
        Map<String, io.swagger.models.Response> responses = operation.getResponses();
        if (responses != null) {
            String[] keys = new String[responses.keySet().size()];
            Arrays.sort(responses.keySet().toArray(keys));
            int code = 0;
            String defaultKey = null;
            for (String key : keys) {
                if (key.startsWith("2")) {
                    defaultKey = key;
                    code = Integer.parseInt(key);
                    break;
                }
                if ("default".equals(key)) {
                    defaultKey = key;
                    code = 200;
                    break;
                }
                if (key.startsWith("3")) {
                    // we use the 3xx responses as defaults
                    defaultKey = key;
                    code = Integer.parseInt(key);
                }
            }

            if (defaultKey != null) {
                ResponseBuilder builder = Response.status(code);
                io.swagger.models.Response response = responses.get(defaultKey);

                if (response.getHeaders() != null && response.getHeaders().size() > 0) {
                    for (String key : response.getHeaders().keySet()) {
                        Property headerProperty = response.getHeaders().get(key);
                        Object output = ExampleBuilder.fromProperty(headerProperty, definitions);
                        if (output instanceof ArrayExample) {
                            output = ((ArrayExample) output).asString();
                        } else if (output instanceof ObjectExample) {
                            LOGGER.debug(
                                    "not serializing output example, only primitives or arrays of primitives are supported");
                        } else {
                            output = ((Example) output).asString();
                        }
                        builder.header(key, output);
                    }
                }

                Map<String, Object> examples = response.getExamples();
                if (examples != null) {
                    for (MediaType mediaType : requestContext.getAcceptableMediaTypes()) {
                        for (String key : examples.keySet()) {
                            if (MediaType.valueOf(key).isCompatible(mediaType)) {
                                builder.entity(examples.get(key)).type(mediaType);

                                return builder.build();
                            }
                        }
                    }
                }

                Object output = ExampleBuilder.fromProperty(response.getSchema(), definitions);

                if (output != null) {
                    ResponseContext resp = new ResponseContext().entity(output);
                    setContentType(requestContext, resp, operation);
                    builder.entity(output);
                    if (resp.getContentType() != null) {
                        // this comes from the operation itself
                        builder.type(resp.getContentType());
                    } else {
                        // get acceptable content types
                        List<EntityProcessor> processors = EntityProcessorFactory.getProcessors();

                        MediaType responseMediaType = null;

                        // take first compatible one
                        for (EntityProcessor processor : processors) {
                            if (responseMediaType != null) {
                                break;
                            }
                            for (MediaType mt : requestContext.getAcceptableMediaTypes()) {
                                LOGGER.debug("checking type " + mt.toString() + " against "
                                        + processor.getClass().getName());
                                if (processor.supports(mt)) {
                                    builder.type(mt);
                                    responseMediaType = mt;
                                    break;
                                }
                            }
                        }

                        if (responseMediaType == null) {
                            // no match based on Accept header, use first processor in list
                            for (EntityProcessor processor : processors) {
                                List<MediaType> supportedTypes = processor.getSupportedMediaTypes();
                                if (supportedTypes.size() > 0) {
                                    builder.type(supportedTypes.get(0));
                                    break;
                                }
                            }
                        }
                    }

                    builder.entity(output);
                }
                return builder.build();
            } else {
                LOGGER.debug("no response type to map to, assume 200");
                code = 200;
            }
            return Response.status(code).build();
        }
        return Response.ok().build();
    } finally {
        for (String key : inputStreams.keySet()) {
            File file = inputStreams.get(key);
            if (file != null) {
                LOGGER.debug("deleting file " + file.getPath());
                file.delete();
            }
        }
    }
}

---

From source file:canreg.client.dataentry.Convert.java

public static boolean importFile(canreg.client.gui.management.CanReg4MigrationInternalFrame.MigrationTask task,
        Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server,
        ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException {
    boolean success = false;

    Set<String> noNeedToLookAtPatientVariables = new TreeSet<String>();

    noNeedToLookAtPatientVariables//from  w w  w.j  a v  a 2s. c o  m
            .add(canreg.common.Tools.toLowerCaseStandardized(io.getPatientIDVariableName()));
    noNeedToLookAtPatientVariables
            .add(canreg.common.Tools.toLowerCaseStandardized(io.getPatientRecordIDVariableName()));

    String firstNameVariableName = io.getFirstNameVariableName();
    String sexVariableName = io.getSexVariableName();

    CSVParser parser = null;

    HashMap mpCodes = new HashMap();

    int numberOfLinesRead = 0;

    Map<String, Integer> nameSexTable = server.getNameSexTables();

    try {
        // Logger.getLogger(Import.class.getName()).log(Level.CONFIG, "Name of the character encoding {0}");

        int numberOfRecordsInFile = canreg.common.Tools.numberOfLinesInFile(file.getAbsolutePath());

        debugOut("Importing data from " + file);

        CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(io.getSeparator());

        parser = CSVParser.parse(file, io.getFileCharset(), format);

        int linesToRead = io.getMaxLines();
        if (linesToRead == -1 || linesToRead > numberOfRecordsInFile) {
            linesToRead = numberOfRecordsInFile;
        }

        for (CSVRecord csvRecord : parser) {
            numberOfLinesRead++;
            // We allow for null tasks...
            boolean needToSavePatientAgain = true;
            int patientDatabaseRecordID = -1;

            if (task != null) {
                if (canreg.client.gui.management.CanReg4MigrationInternalFrame.isPaused) {
                    task.firePropertyChange("paused", false, true);
                }
                if (!canreg.client.gui.management.CanReg4MigrationInternalFrame.isPaused) {
                    task.firePropertyChange("paused", true, false);
                    task.firePropertyChange("progress", (numberOfLinesRead - 1) * 100 / linesToRead,
                            (numberOfLinesRead) * 100 / linesToRead);
                }
            }

            // Build patient part
            Patient patient = new Patient();
            for (int i = 0; i < map.size(); i++) {
                Relation rel = map.get(i);
                if (rel.getDatabaseTableVariableID() >= 0
                        && rel.getDatabaseTableName().equalsIgnoreCase("patient")) {
                    if (rel.getFileColumnNumber() < csvRecord.size()) {
                        if (rel.getVariableType().equalsIgnoreCase("Number")) {
                            if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) {
                                try {
                                    patient.setVariable(rel.getDatabaseVariableName(),
                                            Integer.parseInt(csvRecord.get(rel.getFileColumnNumber())));
                                } catch (NumberFormatException ex) {
                                    Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                                            "Number format error in line: " + (numberOfLinesRead + 1 + 1)
                                                    + ". ",
                                            ex);
                                    success = false;
                                }
                            }
                        } else {
                            patient.setVariable(rel.getDatabaseVariableName(),
                                    StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber())));
                        }
                    } else {
                        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                                "Something wrong with patient part of line " + numberOfLinesRead + ".",
                                new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: "
                                        + rel.getDatabaseVariableName()));
                    }
                }
            }
            // debugOut(patient.toString());

            // Build tumour part
            Tumour tumour = new Tumour();
            for (int i = 0; i < map.size(); i++) {
                Relation rel = map.get(i);
                if (rel.getDatabaseTableVariableID() >= 0
                        && rel.getDatabaseTableName().equalsIgnoreCase("tumour")
                        && rel.getFileColumnNumber() < csvRecord.size()) {
                    if (rel.getFileColumnNumber() < csvRecord.size()) {
                        if (rel.getVariableType().equalsIgnoreCase("Number")) {
                            if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) {
                                try {
                                    tumour.setVariable(rel.getDatabaseVariableName(),
                                            Integer.parseInt(csvRecord.get(rel.getFileColumnNumber())));
                                } catch (NumberFormatException ex) {
                                    Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                                            "Number format error in line: " + (numberOfLinesRead + 1 + 1)
                                                    + ". ",
                                            ex);
                                    success = false;
                                }
                            }
                        } else {
                            tumour.setVariable(rel.getDatabaseVariableName(),
                                    StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber())));
                        }
                    } else {
                        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                                "Something wrong with tumour part of line " + numberOfLinesRead + ".",
                                new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: "
                                        + rel.getDatabaseVariableName()));
                    }
                }
            }

            // Build source part
            Set<Source> sources = Collections.synchronizedSet(new LinkedHashSet<Source>());
            Source source = new Source();
            for (int i = 0; i < map.size(); i++) {
                Relation rel = map.get(i);
                if (rel.getDatabaseTableVariableID() >= 0
                        && rel.getDatabaseTableName().equalsIgnoreCase(Globals.SOURCE_TABLE_NAME)
                        && rel.getFileColumnNumber() < csvRecord.size()) {
                    if (rel.getFileColumnNumber() < csvRecord.size()) {
                        if (rel.getVariableType().equalsIgnoreCase("Number")) {
                            if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) {
                                try {
                                    source.setVariable(rel.getDatabaseVariableName(),
                                            Integer.parseInt(csvRecord.get(rel.getFileColumnNumber())));
                                } catch (NumberFormatException ex) {
                                    Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                                            "Number format error in line: " + (numberOfLinesRead + 1 + 1)
                                                    + ". ",
                                            ex);
                                    success = false;
                                }
                            }
                        } else {
                            source.setVariable(rel.getDatabaseVariableName(),
                                    StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber())));
                        }
                    } else {
                        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                                "Something wrong with source part of line " + numberOfLinesRead + ".",
                                new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: "
                                        + rel.getDatabaseVariableName()));
                    }

                }

            }
            sources.add(source);
            tumour.setSources(sources);

            // debugOut(tumour.toString());
            // add patient to the database
            Object patientID = patient.getVariable(io.getPatientIDVariableName());
            Object patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName());

            if (patientID == null) {
                // save the record to get the new patientID;
                patientDatabaseRecordID = server.savePatient(patient);
                patient = (Patient) server.getRecord(patientDatabaseRecordID, Globals.PATIENT_TABLE_NAME,
                        false);
                patientID = patient.getVariable(io.getPatientIDVariableName());
                patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName());
            }

            if (io.isDataFromPreviousCanReg()) {
                // set update date for the patient the same as for the tumour
                Object updateDate = tumour.getVariable(io.getTumourUpdateDateVariableName());
                patient.setVariable(io.getPatientUpdateDateVariableName(), updateDate);

                // Set the patientID the same as the tumourID initially

                // Object tumourSequence = tumour.getVariable(io.getTumourSequenceVariableName());
                Object tumourSequence = "1";

                String tumourSequenceString = tumourSequence + "";
                while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) {
                    tumourSequenceString = "0" + tumourSequenceString;
                }
                patientRecordID = patientID + "" + tumourSequenceString;

                // If this is a multiple primary tumour...
                String mpCodeString = (String) tumour.getVariable(io.getMultiplePrimaryVariableName());
                if (mpCodeString != null && mpCodeString.length() > 0) {
                    patientID = lookUpPatientID(mpCodeString, patientID, mpCodes);

                    // rebuild sequenceNumber
                    Tumour[] tumours = new Tumour[0];
                    try {
                        tumours = CanRegClientApp.getApplication()
                                .getTumourRecordsBasedOnPatientID(patientID + "", false);
                    } catch (DistributedTableDescriptionException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (UnknownTableException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    }

                    tumourSequenceString = (tumours.length + 1) + "";
                    while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) {
                        tumourSequenceString = "0" + tumourSequenceString;
                    }

                    patientRecordID = patientID + "" + tumourSequenceString;
                    Patient[] oldPatients = null;
                    try {
                        oldPatients = CanRegClientApp.getApplication().getPatientRecordsByID((String) patientID,
                                false);
                    } catch (RemoteException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (SecurityException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (DistributedTableDescriptionException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (RecordLockedException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (SQLException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (UnknownTableException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    }
                    for (Patient oldPatient : oldPatients) {
                        if (!Tools.newRecordContainsNewInfo(patient, oldPatient,
                                noNeedToLookAtPatientVariables)) {
                            needToSavePatientAgain = false;
                            patient = oldPatient;
                            patientRecordID = oldPatient.getVariable(io.getPatientRecordIDVariableName());
                        }
                    }
                }

                Object tumourID = patientRecordID + "" + tumourSequenceString;
                //
                patient.setVariable(io.getPatientIDVariableName(), patientID);
                tumour.setVariable(io.getTumourIDVariablename(), tumourID);
                // And store the record ID

                patient.setVariable(io.getPatientRecordIDVariableName(), patientRecordID);

                // Set the patient ID number on the tumour
                tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID);
                tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID);

                // Set the deprecated flag to 0 - no obsolete records from CR4
                tumour.setVariable(io.getObsoleteTumourFlagVariableName(), "0");
                patient.setVariable(io.getObsoletePatientFlagVariableName(), "0");

            }

            // Set the name in the firstName database
            String sex = (String) patient.getVariable(sexVariableName);
            if (sex != null && sex.length() > 0) {
                Integer sexCode = Integer.parseInt(sex);
                String firstNames = (String) patient.getVariable(firstNameVariableName);
                if (firstNames != null) {
                    String[] firstNamesArray = firstNames.split(" ");
                    for (String firstName : firstNamesArray) {
                        if (firstName != null && firstName.trim().length() > 0) {
                            // here we use the locale specific toUpperCase
                            Integer registeredSexCode = nameSexTable.get(firstName);
                            if (registeredSexCode == null) {
                                NameSexRecord nsr = new NameSexRecord();
                                nsr.setName(firstName);
                                nsr.setSex(sexCode);

                                server.saveNameSexRecord(nsr, false);

                                nameSexTable.put(firstName, sexCode);
                            } else if (registeredSexCode != sexCode) {
                                if (registeredSexCode != 9) {
                                    sexCode = 9;
                                    NameSexRecord nsr = new NameSexRecord();
                                    nsr.setName(firstName);
                                    nsr.setSex(sexCode);
                                    server.saveNameSexRecord(nsr, true);
                                    nameSexTable.remove(firstName);
                                    nameSexTable.put(firstName, sexCode);
                                }
                            }
                        }
                    }
                }
            }

            if (needToSavePatientAgain) {
                if (patientDatabaseRecordID > 0) {
                    server.editPatient(patient);
                } else {
                    patientDatabaseRecordID = server.savePatient(patient);
                }
            }
            if (patient != null && tumour != null) {
                String icd10 = (String) tumour.getVariable(io.getICD10VariableName());
                if (icd10 == null || icd10.trim().length() == 0) {
                    ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication()
                            .performConversions(Converter.ConversionName.ICDO3toICD10, patient, tumour);
                    tumour.setVariable(io.getICD10VariableName(), conversionResult[0].getValue());
                }
            }
            if (tumour.getVariable(io.getPatientIDTumourTableVariableName()) == null) {
                tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID);
            }

            if (tumour.getVariable(io.getPatientRecordIDTumourTableVariableName()) == null) {
                tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID);
            }

            int tumourDatabaseIDNumber = server.saveTumour(tumour);

            if (Thread.interrupted()) {
                //We've been interrupted: no more importing.
                throw new InterruptedException();
            }
        }
        task.firePropertyChange("finished", null, null);
        success = true;
    } catch (IOException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } catch (NumberFormatException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } catch (InterruptedException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                "Interupted on line: " + (numberOfLinesRead + 1) + ". ", ex);
        success = true;
    } catch (IndexOutOfBoundsException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } catch (SQLException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } finally {
        if (parser != null) {
            try {
                parser.close();
            } catch (IOException ex) {
                Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }
    return success;
}

---

From source file:canreg.client.dataentry.Import.java

/**
 *
 * @param task/*from ww w .ja v  a 2 s  .  co  m*/
 * @param doc
 * @param map
 * @param file
 * @param server
 * @param io
 * @return
 * @throws java.sql.SQLException
 * @throws java.rmi.RemoteException
 * @throws canreg.server.database.RecordLockedException
 */
public static boolean importFile(Task<Object, String> task, Document doc,
        List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io)
        throws SQLException, RemoteException, SecurityException, RecordLockedException {
    //public static boolean importFile(canreg.client.gui.management.CanReg4MigrationInternalFrame.MigrationTask task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException {
    boolean success = false;

    Set<String> noNeedToLookAtPatientVariables = new TreeSet<String>();

    noNeedToLookAtPatientVariables.add(io.getPatientIDVariableName());
    noNeedToLookAtPatientVariables.add(io.getPatientRecordIDVariableName());

    String firstNameVariableName = io.getFirstNameVariableName();
    String sexVariableName = io.getSexVariableName();

    CSVParser parser = null;
    CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(io.getSeparator());

    int linesToRead = io.getMaxLines();

    HashMap mpCodes = new HashMap();

    int numberOfLinesRead = 0;

    Map<String, Integer> nameSexTable = server.getNameSexTables();

    try {
        //            FileInputStream fis = new FileInputStream(file);
        //           BufferedReader bsr = new BufferedReader(new InputStreamReader(fis, io.getFileCharset()));

        // Logger.getLogger(Import.class.getName()).log(Level.CONFIG, "Name of the character encoding {0}");
        int numberOfRecordsInFile = canreg.common.Tools.numberOfLinesInFile(file.getAbsolutePath());

        if (linesToRead > 0) {
            linesToRead = Math.min(numberOfRecordsInFile, linesToRead);
        } else {
            linesToRead = numberOfRecordsInFile;
        }

        parser = CSVParser.parse(file, io.getFileCharset(), format);

        for (CSVRecord csvRecord : parser) {
            numberOfLinesRead++;
            // We allow for null tasks...
            boolean needToSavePatientAgain = true;
            int patientDatabaseRecordID = -1;

            if (task != null) {
                task.firePropertyChange("progress", (numberOfLinesRead - 1) * 100 / linesToRead,
                        (numberOfLinesRead) * 100 / linesToRead);
            }

            // Build patient part
            Patient patient = new Patient();
            for (int i = 0; i < map.size(); i++) {
                Relation rel = map.get(i);
                if (rel.getDatabaseTableVariableID() >= 0
                        && rel.getDatabaseTableName().equalsIgnoreCase("patient")) {
                    if (rel.getFileColumnNumber() < csvRecord.size()) {
                        if (rel.getVariableType().equalsIgnoreCase("Number")) {
                            if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) {
                                try {
                                    patient.setVariable(rel.getDatabaseVariableName(),
                                            Integer.parseInt(csvRecord.get(rel.getFileColumnNumber())));
                                } catch (NumberFormatException ex) {
                                    Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                                            "Number format error in line: " + (numberOfLinesRead + 1 + 1)
                                                    + ". ",
                                            ex);
                                    success = false;
                                }
                            }
                        } else {
                            patient.setVariable(rel.getDatabaseVariableName(),
                                    StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber())));
                        }
                    } else {
                        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                                "Something wrong with patient part of line " + numberOfLinesRead + ".",
                                new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: "
                                        + rel.getDatabaseVariableName()));
                    }
                }
            }
            // debugOut(patient.toString());

            // Build tumour part
            Tumour tumour = new Tumour();
            for (canreg.client.dataentry.Relation rel : map) {
                if (rel.getDatabaseTableVariableID() >= 0
                        && rel.getDatabaseTableName().equalsIgnoreCase("tumour")) {
                    if (rel.getFileColumnNumber() < csvRecord.size()) {
                        if (rel.getVariableType().equalsIgnoreCase("Number")) {
                            if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) {
                                try {
                                    tumour.setVariable(rel.getDatabaseVariableName(),
                                            Integer.parseInt(csvRecord.get(rel.getFileColumnNumber())));
                                } catch (NumberFormatException ex) {
                                    Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                                            "Number format error in line: " + (numberOfLinesRead + 1 + 1)
                                                    + ". ",
                                            ex);
                                    success = false;
                                }
                            }
                        } else {
                            tumour.setVariable(rel.getDatabaseVariableName(),
                                    StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber())));
                        }
                    } else {
                        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                                "Something wrong with tumour part of line " + numberOfLinesRead + ".",
                                new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: "
                                        + rel.getDatabaseVariableName()));
                    }
                }
            }

            // Build source part
            Set<Source> sources = Collections.synchronizedSet(new LinkedHashSet<Source>());
            Source source = new Source();
            for (canreg.client.dataentry.Relation rel : map) {
                if (rel.getDatabaseTableVariableID() >= 0
                        && rel.getDatabaseTableName().equalsIgnoreCase(Globals.SOURCE_TABLE_NAME)) {
                    if (rel.getFileColumnNumber() < csvRecord.size()) {
                        if (rel.getVariableType().equalsIgnoreCase("Number")) {
                            if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) {
                                try {
                                    source.setVariable(rel.getDatabaseVariableName(),
                                            Integer.parseInt(csvRecord.get(rel.getFileColumnNumber())));
                                } catch (NumberFormatException ex) {
                                    Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                                            "Number format error in line: " + (numberOfLinesRead + 1 + 1)
                                                    + ". ",
                                            ex);
                                    success = false;
                                }
                            }
                        } else {
                            source.setVariable(rel.getDatabaseVariableName(),
                                    StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber())));
                        }
                    } else {
                        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                                "Something wrong with source part of line " + numberOfLinesRead + ".",
                                new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: "
                                        + rel.getDatabaseVariableName()));
                    }

                }
            }
            sources.add(source);
            tumour.setSources(sources);

            // debugOut(tumour.toString());
            // add patient to the database
            Object patientID = patient.getVariable(io.getPatientIDVariableName());
            Object patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName());

            if (patientID == null) {
                // save the record to get the new patientID;
                patientDatabaseRecordID = server.savePatient(patient);
                patient = (Patient) server.getRecord(patientDatabaseRecordID, Globals.PATIENT_TABLE_NAME,
                        false);
                patientID = patient.getVariable(io.getPatientIDVariableName());
                patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName());
            }

            if (io.isDataFromPreviousCanReg()) {
                // set update date for the patient the same as for the tumour
                Object updateDate = tumour.getVariable(io.getTumourUpdateDateVariableName());
                patient.setVariable(io.getPatientUpdateDateVariableName(), updateDate);

                // Set the patientID the same as the tumourID initially
                // Object tumourSequence = tumour.getVariable(io.getTumourSequenceVariableName());
                Object tumourSequence = "1";

                String tumourSequenceString = tumourSequence + "";
                while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) {
                    tumourSequenceString = "0" + tumourSequenceString;
                }
                patientRecordID = patientID + "" + tumourSequenceString;

                // If this is a multiple primary tumour...
                String mpCodeString = (String) tumour.getVariable(io.getMultiplePrimaryVariableName());
                if (mpCodeString != null && mpCodeString.length() > 0) {
                    patientID = lookUpPatientID(mpCodeString, patientID, mpCodes);

                    // rebuild sequenceNumber
                    Tumour[] tumours = new Tumour[0];
                    try {
                        tumours = CanRegClientApp.getApplication()
                                .getTumourRecordsBasedOnPatientID(patientID + "", false);
                    } catch (DistributedTableDescriptionException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (UnknownTableException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    }

                    tumourSequenceString = (tumours.length + 1) + "";
                    while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) {
                        tumourSequenceString = "0" + tumourSequenceString;
                    }

                    patientRecordID = patientID + "" + tumourSequenceString;
                    Patient[] oldPatients = null;
                    try {
                        oldPatients = CanRegClientApp.getApplication().getPatientRecordsByID((String) patientID,
                                false);
                    } catch (RemoteException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (SecurityException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (DistributedTableDescriptionException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (RecordLockedException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (SQLException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    } catch (UnknownTableException ex) {
                        Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
                    }
                    for (Patient oldPatient : oldPatients) {
                        if (!Tools.newRecordContainsNewInfo(patient, oldPatient,
                                noNeedToLookAtPatientVariables)) {
                            needToSavePatientAgain = false;
                            patient = oldPatient;
                            patientRecordID = oldPatient.getVariable(io.getPatientRecordIDVariableName());
                        }
                    }
                }

                Object tumourID = patientRecordID + "" + tumourSequenceString;
                //
                patient.setVariable(io.getPatientIDVariableName(), patientID);
                tumour.setVariable(io.getTumourIDVariablename(), tumourID);
                // And store the record ID

                patient.setVariable(io.getPatientRecordIDVariableName(), patientRecordID);

                // Set the patient ID number on the tumour
                tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID);
                tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID);

                // Set the deprecated flag to 0 - no obsolete records from CR4
                tumour.setVariable(io.getObsoleteTumourFlagVariableName(), "0");
                patient.setVariable(io.getObsoletePatientFlagVariableName(), "0");

            }

            // Set the name in the firstName database
            String sex = (String) patient.getVariable(sexVariableName);
            if (sex != null && sex.length() > 0) {
                Integer sexCode = Integer.parseInt(sex);
                String firstNames = (String) patient.getVariable(firstNameVariableName);
                if (firstNames != null) {
                    String[] firstNamesArray = firstNames.split(" ");
                    for (String firstName : firstNamesArray) {
                        if (firstName != null && firstName.trim().length() > 0) {
                            // here we use the locale specific toUpperCase
                            Integer registeredSexCode = nameSexTable.get(firstName);
                            if (registeredSexCode == null) {
                                NameSexRecord nsr = new NameSexRecord();
                                nsr.setName(firstName);
                                nsr.setSex(sexCode);

                                server.saveNameSexRecord(nsr, false);

                                nameSexTable.put(firstName, sexCode);
                            } else if (registeredSexCode != sexCode) {
                                if (registeredSexCode != 9) {
                                    sexCode = 9;
                                    NameSexRecord nsr = new NameSexRecord();
                                    nsr.setName(firstName);
                                    nsr.setSex(sexCode);
                                    server.saveNameSexRecord(nsr, true);
                                    nameSexTable.remove(firstName);
                                    nameSexTable.put(firstName, sexCode);
                                }
                            }
                        }
                    }
                }
            }

            if (needToSavePatientAgain) {
                if (patientDatabaseRecordID > 0) {
                    server.editPatient(patient);
                } else {
                    patientDatabaseRecordID = server.savePatient(patient);
                }
            }
            if (patient != null && tumour != null) {
                String icd10 = (String) tumour.getVariable(io.getICD10VariableName());
                if (icd10 == null || icd10.trim().length() == 0) {
                    ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication()
                            .performConversions(Converter.ConversionName.ICDO3toICD10, patient, tumour);
                    tumour.setVariable(io.getICD10VariableName(), conversionResult[0].getValue());
                }
                String iccc = (String) tumour.getVariable(io.getICCCVariableName());
                if (iccc == null || iccc.trim().length() == 0) {
                    ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication()
                            .performConversions(Converter.ConversionName.ICDO3toICCC3, patient, tumour);
                    tumour.setVariable(io.getICCCVariableName(), conversionResult[0].getValue());
                }
            }
            if (tumour.getVariable(io.getPatientIDTumourTableVariableName()) == null) {
                tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID);
            }

            if (tumour.getVariable(io.getPatientRecordIDTumourTableVariableName()) == null) {
                tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID);
            }

            int tumourDatabaseIDNumber = server.saveTumour(tumour);

            if (Thread.interrupted()) {
                //We've been interrupted: no more importing.
                throw new InterruptedException();
            }
        }
        task.firePropertyChange("finished", null, null);
        success = true;
    } catch (IOException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } catch (NumberFormatException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } catch (InterruptedException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.INFO,
                "Interupted on line: " + (numberOfLinesRead + 1) + ". ", ex);
        success = true;
    } catch (IndexOutOfBoundsException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } catch (SQLException ex) {
        Logger.getLogger(Import.class.getName()).log(Level.SEVERE,
                "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex);
        success = false;
    } finally {
        if (parser != null) {
            try {
                parser.close();
            } catch (IOException ex) {
                Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
    }

    return success;
}

---

From source file:com.amazonaws.services.dynamodbv2.online.index.integration.tests.ViolationCorrectionTest.java

/**
 * Iterates through detection output file: first leave updates blank based on missing updates per key. 
 * Once it has reached the missing update number, it removes the expected gsi values as per the specified 'missingGsiExpectedHashValues'.
 * Note that once blank number is reached, it also starts adding updates. 
 * It then iterates over the rows again and adds values for Yes/No/Invalid in the delete column.
 * It returns all error records, if present. If not, it returns all records.
 *///w  ww. ja  va 2s . c  om
private static List<List<String>> createCorrectionFile(final String detectionFile, final String correctionFile,
        final String gsiHashKeyName, final String gsiHashKeyType, final String gsiRangeKeyName,
        final String gsiRangeKeyType, final Map<String, String> tableHashToNewGsiHashValueMap,
        final Map<String, String> tableHashToNewGsiRangeValueMap, final int missingUpdatesPerKey,
        final int missingGsiExpectedHashValues, final int invalidValuesForDelete, final int numOfYesForDelete,
        final int numOfNoForDelete) throws IOException {

    List<List<String>> errorRecords = null;
    List<List<String>> allRecords = null;

    BufferedReader br = null;
    BufferedWriter bw = null;
    CSVParser parser = null;
    CSVPrinter csvPrinter = null;
    try {
        br = new BufferedReader(new FileReader(new File(detectionFile)));
        bw = new BufferedWriter(new FileWriter(new File(correctionFile)));
        parser = new CSVParser(br, TestUtils.csvFormat);
        csvPrinter = new CSVPrinter(bw, TestUtils.csvFormat);
        List<CSVRecord> detectorRecords = parser.getRecords();

        int hashMissingUpdates = 0;
        int rangeMissingUpdates = 0;
        int missingGsiExpectedHashValuesCurrent = 0;

        // Print Header
        Map<String, Integer> header = parser.getHeaderMap();
        csvPrinter.printRecord(header.keySet());

        allRecords = new ArrayList<List<String>>();
        for (CSVRecord csvRecord : detectorRecords) {
            List<String> newRecord = new ArrayList<String>();
            String tableHashKeyRecorded = csvRecord.get(ViolationRecord.TABLE_HASH_KEY);

            String hashKeyViolationType = null;
            if (gsiHashKeyName != null) {
                hashKeyViolationType = csvRecord.get(ViolationRecord.GSI_HASH_KEY_VIOLATION_TYPE);
            }
            String rangeKeyViolationType = null;
            if (gsiRangeKeyName != null) {
                rangeKeyViolationType = csvRecord.get(ViolationRecord.GSI_RANGE_KEY_VIOLATION_TYPE);
            }

            for (int i = 0; i < csvRecord.size(); i++) {
                newRecord.add(i, csvRecord.get(i));
            }

            String newGsiVal = null;
            if (hashKeyViolationType != null && (hashKeyViolationType.equals("Size Violation")
                    || hashKeyViolationType.equals("Type Violation"))) {
                if (hashMissingUpdates < missingUpdatesPerKey) {
                    allRecords.add(newRecord);
                    hashMissingUpdates++;
                    continue;
                }
                //Remove expected hash Values
                if (missingGsiExpectedHashValuesCurrent < missingGsiExpectedHashValues) {
                    newRecord.remove((int) header.get(ViolationRecord.GSI_HASH_KEY));
                    newRecord.add(header.get(ViolationRecord.GSI_HASH_KEY), "");
                    missingGsiExpectedHashValuesCurrent++;
                }

                newRecord.remove((int) header.get(ViolationRecord.GSI_HASH_KEY_UPDATE_VALUE));
                newGsiVal = getNewValue(gsiHashKeyType, 4 /*length*/);
                newRecord.add(header.get(ViolationRecord.GSI_HASH_KEY_UPDATE_VALUE), newGsiVal);
                tableHashToNewGsiHashValueMap.put(tableHashKeyRecorded, newGsiVal);
            }

            if (rangeKeyViolationType != null && (rangeKeyViolationType.equals("Size Violation")
                    || rangeKeyViolationType.equals("Type Violation"))) {
                if (rangeMissingUpdates < missingUpdatesPerKey) {
                    allRecords.add(newRecord);
                    rangeMissingUpdates++;
                    continue;
                }

                newRecord.remove(header.get(ViolationRecord.GSI_RANGE_KEY_UPDATE_VALUE));
                newGsiVal = getNewValue(gsiRangeKeyType, 4 /*length*/);
                newRecord.add(header.get(ViolationRecord.GSI_RANGE_KEY_UPDATE_VALUE), newGsiVal);
                tableHashToNewGsiRangeValueMap.put(tableHashKeyRecorded, newGsiVal);
            }
            allRecords.add(newRecord);
        }

        // Add 'Y' or 'N' for delete column
        if (numOfNoForDelete > 0 || numOfYesForDelete > 0 || invalidValuesForDelete > 0) {
            errorRecords = new ArrayList<List<String>>();
            int numOfYesAdded = 0;
            int numOfNoAdded = 0;
            int numOfInvalids = 0;
            for (List<String> record : allRecords) {
                if (numOfInvalids < invalidValuesForDelete) {
                    record.remove(header.get(ViolationRecord.GSI_CORRECTION_DELETE_BLANK));
                    record.add(header.get(ViolationRecord.GSI_CORRECTION_DELETE_BLANK), "xx");
                    numOfInvalids++;
                    errorRecords.add(record);
                    continue;
                }

                if (numOfYesAdded < numOfYesForDelete) {
                    record.remove(header.get(ViolationRecord.GSI_CORRECTION_DELETE_BLANK));
                    record.add(header.get(ViolationRecord.GSI_CORRECTION_DELETE_BLANK), "Y");
                    numOfYesAdded++;
                    continue;
                }

                if (numOfNoAdded < numOfNoForDelete) {
                    record.remove(header.get(ViolationRecord.GSI_CORRECTION_DELETE_BLANK));
                    record.add(header.get(ViolationRecord.GSI_CORRECTION_DELETE_BLANK), "N");
                    numOfNoAdded++;
                    continue;
                }
            }
        }

        // Add all records to file
        csvPrinter.printRecords(allRecords);
    } finally {
        br.close();
        bw.close();
        parser.close();
        csvPrinter.close();
    }

    if (errorRecords != null)
        return errorRecords;
    else
        return allRecords;
}

---

From source file:io.dockstore.client.cli.nested.AbstractEntryClient.java

private void launchCwl(String entry, final List<String> args) throws ApiException, IOException {
    boolean isLocalEntry = false;
    if (args.contains("--local-entry")) {
        isLocalEntry = true;// w w  w.  java2  s .c o m
    }

    final String yamlRun = optVal(args, "--yaml", null);
    String jsonRun = optVal(args, "--json", null);
    final String csvRuns = optVal(args, "--tsv", null);

    if (!(yamlRun != null ^ jsonRun != null ^ csvRuns != null)) {
        errorMessage("One of  --json, --yaml, and --tsv is required", CLIENT_ERROR);
    }

    final File tempDir = Files.createTempDir();
    File tempCWL;
    if (!isLocalEntry) {
        tempCWL = File.createTempFile("temp", ".cwl", tempDir);
    } else {
        tempCWL = new File(entry);
    }

    if (!isLocalEntry) {
        final SourceFile cwlFromServer = getDescriptorFromServer(entry, "cwl");
        Files.write(cwlFromServer.getContent(), tempCWL, StandardCharsets.UTF_8);
        downloadDescriptors(entry, "cwl", tempDir);
    }
    jsonRun = convertYamlToJson(yamlRun, jsonRun);

    final Gson gson = io.cwl.avro.CWL.getTypeSafeCWLToolDocument();
    if (jsonRun != null) {
        // if the root document is an array, this indicates multiple runs
        JsonParser parser = new JsonParser();
        final JsonElement parsed = parser
                .parse(new InputStreamReader(new FileInputStream(jsonRun), StandardCharsets.UTF_8));
        if (parsed.isJsonArray()) {
            final JsonArray asJsonArray = parsed.getAsJsonArray();
            for (JsonElement element : asJsonArray) {
                final String finalString = gson.toJson(element);
                final File tempJson = File.createTempFile("parameter", ".json", Files.createTempDir());
                FileUtils.write(tempJson, finalString, StandardCharsets.UTF_8);
                final LauncherCWL cwlLauncher = new LauncherCWL(getConfigFile(), tempCWL.getAbsolutePath(),
                        tempJson.getAbsolutePath());
                if (this instanceof WorkflowClient) {
                    cwlLauncher.run(Workflow.class);
                } else {
                    cwlLauncher.run(CommandLineTool.class);
                }
            }
        } else {
            final LauncherCWL cwlLauncher = new LauncherCWL(getConfigFile(), tempCWL.getAbsolutePath(),
                    jsonRun);
            if (this instanceof WorkflowClient) {
                cwlLauncher.run(Workflow.class);
            } else {
                cwlLauncher.run(CommandLineTool.class);
            }
        }
    } else if (csvRuns != null) {
        final File csvData = new File(csvRuns);
        try (CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8,
                CSVFormat.DEFAULT.withDelimiter('\t').withEscape('\\').withQuoteMode(QuoteMode.NONE))) {
            // grab header
            final Iterator<CSVRecord> iterator = parser.iterator();
            final CSVRecord headers = iterator.next();
            // ignore row with type information
            iterator.next();
            // process rows
            while (iterator.hasNext()) {
                final CSVRecord csvRecord = iterator.next();
                final File tempJson = File.createTempFile("temp", ".json", Files.createTempDir());
                StringBuilder buffer = new StringBuilder();
                buffer.append("{");
                for (int i = 0; i < csvRecord.size(); i++) {
                    buffer.append("\"").append(headers.get(i)).append("\"");
                    buffer.append(":");
                    // if the type is an array, just pass it through
                    buffer.append(csvRecord.get(i));

                    if (i < csvRecord.size() - 1) {
                        buffer.append(",");
                    }
                }
                buffer.append("}");
                // prettify it
                JsonParser prettyParser = new JsonParser();
                JsonObject json = prettyParser.parse(buffer.toString()).getAsJsonObject();
                final String finalString = gson.toJson(json);

                // write it out
                FileUtils.write(tempJson, finalString, StandardCharsets.UTF_8);

                // final String stringMapAsString = gson.toJson(stringMap);
                // Files.write(stringMapAsString, tempJson, StandardCharsets.UTF_8);
                final LauncherCWL cwlLauncher = new LauncherCWL(this.getConfigFile(), tempCWL.getAbsolutePath(),
                        tempJson.getAbsolutePath());
                if (this instanceof WorkflowClient) {
                    cwlLauncher.run(Workflow.class);
                } else {
                    cwlLauncher.run(CommandLineTool.class);
                }
            }
        }
    } else {
        errorMessage("Missing required parameters, one of  --json or --tsv is required", CLIENT_ERROR);
    }

}

---

From source file:com.amazonaws.services.dynamodbv2.online.index.integration.tests.ViolationCorrectionTest.java

/**
 * Validates the output of violation correction.
 *///  w w w .  j  a v a 2  s . c  om
private void validateCorrectionOutput(String correctionOutputFile, List<List<String>> errorRecords)
        throws IOException {
    BufferedReader br = null;
    CSVParser parser = null;
    try {
        br = new BufferedReader(new FileReader(new File(correctionOutputFile)));
        parser = new CSVParser(br, TestUtils.csvFormat);
        List<CSVRecord> records = parser.getRecords();

        Assert.assertEquals("Error record count does not match", errorRecords.size(), records.size());
        for (CSVRecord record : records) {
            boolean foundError = false;
            List<String> readRecord = new ArrayList<String>();
            for (int i = 0; i < record.size(); i++) {
                if (record.get(i).equals(record.get(ViolationRecord.GSI_VALUE_UPDATE_ERROR))) {
                    foundError = true;
                    continue;
                } else {
                    readRecord.add(record.get(i));
                }
            }
            Assert.assertTrue("Error column not found", foundError);
            Assert.assertTrue("Unexpected record read from correction output",
                    errorRecords.contains(readRecord));
            errorRecords.remove(readRecord);
        }
    } finally {
        br.close();
        parser.close();
    }
}

---

From source file:edu.nyu.vida.data_polygamy.utils.FrameworkUtils.java

public static String[] splitStr(String val) throws IOException {

    CSVParser parser = new CSVParser(new StringReader(val), CSVFormat.DEFAULT);
    CSVRecord record = parser.getRecords().get(0);
    Iterator<String> valuesIt = record.iterator();
    String[] input = new String[record.size()];
    int i = 0;//from  ww  w  .j  av  a 2 s.  co m
    while (valuesIt.hasNext()) {
        input[i] = valuesIt.next();
        i++;
    }
    parser.close();
    return input;
}

---

From source file:com.adobe.aem.demo.communities.Loader.java

private static void doDecorate(String hostname, String altport, String location, CSVRecord record,
        String analytics) {//from  ww  w .j a v a 2 s . co  m

    // Getting the JSON view of the resource
    String resourceJson = Loader.doGet(hostname, altport, location + ".social.json", "admin", "admin", null);
    logger.debug("JSON view of the resource is: " + resourceJson);

    // Generating random ratings and comments for the resource for each of the enrolled users
    try {

        JSONObject resourceJsonObject = new JSONObject(resourceJson);

        String resourceRatingsEndpoint = resourceJsonObject.getString("ratingsEndPoint");
        String resourceCommentsEndpoint = resourceJsonObject.getString("commentsEndPoint");
        String resourceID = resourceJsonObject.getString("id");
        String resourceType = resourceJsonObject.getJSONObject("assetProperties").getString("type");
        String referer = "http://localhost:" + altport + "/content/sites/" + record.get(RESOURCE_INDEX_SITE)
                + "/en"
                + (record.get(RESOURCE_INDEX_FUNCTION).length() > 0
                        ? ("/" + record.get(RESOURCE_INDEX_FUNCTION))
                        : "")
                + ".resource.html" + resourceID;

        logger.debug("Resource Ratings Endpoint: " + resourceRatingsEndpoint);
        logger.debug("Resource Comments Endpoint: " + resourceCommentsEndpoint);
        logger.debug("Resource Type: " + resourceType);
        logger.debug("Resource ID: " + resourceID);
        logger.debug("Referer: " + referer);

        // Looking for the list of enrolled users
        for (int i = 0; i < record.size() - 1; i = i + 1) {

            if (record.get(i) != null && record.get(i + 1) != null && record.get(i).equals("deltaList")) {

                JSONObject enrolledJsonObject = new JSONObject(record.get(i + 1));
                Iterator<?> iter = enrolledJsonObject.keys();
                while (iter.hasNext()) {

                    String key = (String) iter.next();
                    logger.debug("New Resource Enrollee: " + key);

                    // Getting information about this enrollee (user or group?)
                    List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>();
                    nameValuePairs.add(new BasicNameValuePair("filter",
                            "[{\"operation\":\"like\",\"rep:principalName\":\"" + key + "\"}]"));
                    String list = Loader.doGet(hostname, altport,
                            "/libs/social/console/content/content/userlist.social.0.10.json", "admin", "admin",
                            nameValuePairs);

                    logger.debug(list);

                    JSONArray jsonArray = new JSONObject(list).getJSONArray("items");
                    if (jsonArray.length() == 1) {

                        JSONObject jsonObject = jsonArray.getJSONObject(0);
                        String jsonElement = jsonObject.getString("type");

                        if (jsonElement != null && jsonElement.equals("user")) {

                            // Always generating a page view event
                            if (Math.random() < 0.90)
                                doAnalytics(analytics, "event1", referer, resourceID, resourceType);

                            // Sometimes generating a video view event
                            if (Math.random() < 0.75 && resourceType.equals("video/mp4"))
                                doAnalytics(analytics, "event2", referer, resourceID, resourceType);

                            // Posting ratings and comments
                            if (Math.random() < 0.50)
                                doRatings(hostname, altport, key, resourceRatingsEndpoint, referer, resourceID,
                                        resourceType, analytics);
                            if (Math.random() < 0.35)
                                doComments(hostname, altport, key, resourceCommentsEndpoint, referer,
                                        resourceID, resourceType, analytics);

                        } else {

                            logger.debug("Enrollee is a group :" + key);
                            List<NameValuePair> groupNameValuePairs = new ArrayList<NameValuePair>();
                            groupNameValuePairs.add(new BasicNameValuePair("groupId", key));
                            groupNameValuePairs.add(new BasicNameValuePair("includeSubGroups", "true"));
                            String memberList = Loader.doGet(hostname, altport,
                                    "/content/community-components/en/communitygroupmemberlist/jcr:content/content/communitygroupmember.social.0.100.json",
                                    "admin", "admin", groupNameValuePairs);

                            JSONArray memberJsonArray = new JSONObject(memberList).getJSONArray("items");
                            for (int j = 0; j < memberJsonArray.length(); j++) {
                                JSONObject memberJsonObject = memberJsonArray.getJSONObject(j);
                                String email = memberJsonObject.getString("authorizableId");
                                logger.debug("New group member for decoration: " + email);
                                if (email != null) {

                                    // Always generating a page view event
                                    if (Math.random() < 0.90)
                                        doAnalytics(analytics, "event1", referer, resourceID, "video/mp4");

                                    // Sometimes generating a video view event
                                    if (Math.random() < 0.75 && resourceType.equals("video/mp4"))
                                        doAnalytics(analytics, "event2", referer, resourceID, resourceType);

                                    if (Math.random() < 0.50)
                                        doRatings(hostname, altport, email, resourceRatingsEndpoint, referer,
                                                resourceID, resourceType, analytics);
                                    if (Math.random() < 0.35)
                                        doComments(hostname, altport, email, resourceCommentsEndpoint, referer,
                                                resourceID, resourceType, analytics);
                                }

                            } // For each group member

                        } // If user or group

                    } // If there's a principal name

                } // For each enrollee

                break; // only one possible deltaList attribute for resource and learning paths

            }

        }

    } catch (Exception e) {

        logger.error(e.getMessage());

    }

}

---

From source file:com.adobe.aem.demomachine.communities.Loader.java

private static List<NameValuePair> buildNVP(String hostname, String port, String adminPassword, String path,
        CSVRecord record, int start) {

    List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>();
    List<String> alreadyLoaded = new ArrayList<String>();

    nameValuePairs.add(new BasicNameValuePair("_charset_", "UTF-8"));

    for (int i = start; i < record.size() - 1; i = i + 2) {

        if (record.get(i) != null && record.get(i + 1) != null && record.get(i).length() > 0) {

            // We have a non String hint to pass to the POST Servlet
            String name = record.get(i);
            String value = record.get(i + 1);
            if (value.equals("TRUE")) {
                value = "true";
            }/*from  w  w w  . ja  va 2  s  .  c o m*/
            if (value.equals("FALSE")) {
                value = "false";
            }

            // If it's a reference to a resource, let's make sure it's available first
            if (name.startsWith("cq:cloudserviceconfigs")
                    && !isResourceAvailable(hostname, port, adminPassword, value)) {
                logger.warn("Resource " + value + " is not available");
                continue;
            }

            // We are adding to an existing property supporting multiple values (might not exist yet)
            int addition = name.indexOf("+");
            if (addition > 0 && path != null) {

                name = name.substring(0, addition);

                if (!alreadyLoaded.contains(name)) {
                    alreadyLoaded.add(name);
                    // Getting the existing values
                    String existingValues = doGet(hostname, port, path + ".json", "admin", adminPassword, null);
                    try {
                        JSONObject propertyJson = new JSONObject(existingValues.trim());
                        Iterator<?> keys = propertyJson.keys();
                        while (keys.hasNext()) {

                            String key = (String) keys.next();
                            if (name.startsWith(key)) {
                                JSONArray propertyList = (JSONArray) propertyJson.get(key);
                                for (int j = 0; j < propertyList.length(); j++) {
                                    String propertyValue = (String) propertyList.get(j);
                                    nameValuePairs.add(new BasicNameValuePair(name, propertyValue));
                                }
                            }

                        }

                    } catch (Exception e) {
                        logger.error(e.getMessage());
                    }
                }

                // Indicating it's a property with multiple values
                BasicNameValuePair bvHint = new BasicNameValuePair(name + "@TypeHint", "String[]");
                if (!nameValuePairs.contains(bvHint)) {
                    nameValuePairs.add(bvHint);
                }

                BasicNameValuePair bvOption = new BasicNameValuePair(name, value);
                if (!nameValuePairs.contains(bvOption)) {
                    nameValuePairs.add(bvOption);
                }

                continue;

            }

            // We default to String unless specified otherwise
            int hint = name.indexOf("@");
            if (hint > 0) {
                nameValuePairs.add(new BasicNameValuePair(name.substring(0, hint) + "@TypeHint",
                        name.substring(1 + hint)));
                name = name.substring(0, hint);
            } else {
                nameValuePairs.add(new BasicNameValuePair(name + "@TypeHint", "String"));
            }

            // We have multiple values to pass to the POST servlet, e.g. for a String[]
            int multiple = value.indexOf("|");
            if (multiple > 0) {
                List<String> values = null;
                if (value.indexOf("~") > 0) {
                    values = Arrays.asList(value.split("~", -1));
                } else {
                    values = Arrays.asList(value.split("\\|", -1));
                }
                for (String currentValue : values) {
                    nameValuePairs.add(new BasicNameValuePair(name, currentValue));
                }
            } else {
                nameValuePairs.add(new BasicNameValuePair(name, value));
            }

        }

    }

    return nameValuePairs;

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• »