Example usage for java.io StreamTokenizer StreamTokenizer

Introduction

In this page you can find the example usage for java.io StreamTokenizer StreamTokenizer.

Prototype

public StreamTokenizer(Reader r) 

Source Link

Document

Create a tokenizer that parses the given character stream.

Usage

From source file:keel.Algorithms.ImbalancedClassification.CSMethods.C45CS.C45CS.java

/** Constructor.
 *
 * @param paramFile      The parameters file.
 *
 * @throws Exception   If the algorithm cannot be executed.
 *//*www .  j ava 2 s.  c  o  m*/
public C45CS(String paramFile) throws Exception {
    try {

        // starts the time
        long startTime = System.currentTimeMillis();

        /* Sets the options of the execution from text file*/
        StreamTokenizer tokenizer = new StreamTokenizer(new BufferedReader(new FileReader(paramFile)));
        initTokenizer(tokenizer);
        setOptions(tokenizer);

        /* Sets the options from XML file */
        /** para commons.configuration
         XMLConfiguration config = new XMLConfiguration(paramFile);
           setOptions( config );
         */

        /* Initializes the dataset. */
        modelDataset = new Dataset(modelFileName, true);
        trainDataset = new Dataset(trainFileName, false);
        testDataset = new Dataset(testFileName, false);

        priorsProbabilities = new double[modelDataset.numClasses()];
        priorsProbabilities();
        marginCounts = new double[marginResolution + 1];

        // generate the tree
        generateTree(modelDataset);

        printTrain();
        printTest();
        printResult();
    } catch (Exception e) {
        System.err.println(e.getMessage());
        System.exit(-1);
    }
}

---

From source file:keel.Algorithms.Preprocess.Basic.C45.C45.java

/** Constructor 
 *
 * @param paramFile      The parameters file.
 *
 * @throws Exception   If the algorithm cannot be executed.
 *//*from w  w  w . java  2 s.c  om*/
public C45(String paramFile) throws Exception {
    try {

        // starts the time
        long startTime = System.currentTimeMillis();

        /* Sets the options of the execution from text file*/
        StreamTokenizer tokenizer = new StreamTokenizer(new BufferedReader(new FileReader(paramFile)));
        initTokenizer(tokenizer);
        setOptions(tokenizer);

        /* Sets the options from XML file */
        /** para commons.configuration
         XMLConfiguration config = new XMLConfiguration(paramFile);
           setOptions( config );
         */

        /* Initializes the dataset. */
        modelDataset = new Dataset(modelFileName, true);
        trainDataset = new Dataset(trainFileName, false);
        testDataset = new Dataset(testFileName, false);

        priorsProbabilities = new double[modelDataset.numClasses()];
        priorsProbabilities();
        marginCounts = new double[marginResolution + 1];

    } catch (Exception e) {
        System.err.println(e.getMessage());
        System.exit(-1);
    }
}

---

From source file:SimpleCalcStreamTok.java

/** Construct from an existing Reader */
public SimpleCalcStreamTok(Reader rdr) throws IOException {
    tf = new StreamTokenizer(rdr);
    // Control the input character set:
    tf.slashSlashComments(true); // treat "//" as comments
    tf.ordinaryChar('-'); // used for subtraction
    tf.ordinaryChar('/'); // used for division

    s = new Stack();
}

---

From source file:com.feilong.core.bean.ConvertUtilTest.java

/**
 * TestConvertUtilTest.// ww  w . j  a  v  a 2s  .  c om
 * 
 * @throws IOException
 */
@Test
public void testConvertUtilTest5() throws IOException {
    StreamTokenizer streamTokenizer = new StreamTokenizer(new StringReader("abaBc^babac^cb//ab/*test*/"));
    streamTokenizer.whitespaceChars('^', '^'); // Set the delimiters
    streamTokenizer.lowerCaseMode(true);

    streamTokenizer.slashSlashComments(false);
    streamTokenizer.slashStarComments(false);
    // Split comma-delimited tokens into a List
    List<String> list = new ArrayList<String>();
    while (true) {
        int ttype = streamTokenizer.nextToken();
        if ((ttype == StreamTokenizer.TT_WORD) || (ttype > 0)) {
            if (streamTokenizer.sval != null) {
                list.add(streamTokenizer.sval);
            }
        } else if (ttype == StreamTokenizer.TT_EOF) {
            break;
        }
    }

    LOGGER.debug(JsonUtil.format(list));
}

---

From source file:net.duckling.ddl.service.render.dml.ParseHtmlImg.java

public Map parseArgs(String argstring) throws IOException {
    HashMap<String, String> arglist = new HashMap<String, String>();

    ////from   www.  ja v  a2  s  .co m
    //  Protection against funny users.
    //
    if (argstring == null) {
        return arglist;
    }

    StringReader in = new StringReader(argstring);
    StreamTokenizer tok = new StreamTokenizer(in);
    int type;

    String param = null;
    String value = null;

    tok.eolIsSignificant(true);

    boolean potentialEmptyLine = false;
    boolean quit = false;

    while (!quit) {
        String s;

        type = tok.nextToken();

        switch (type) {
        case StreamTokenizer.TT_EOF:
            quit = true;
            s = null;
            break;

        case StreamTokenizer.TT_WORD:
            s = tok.sval;
            potentialEmptyLine = false;
            break;

        case StreamTokenizer.TT_EOL:
            quit = potentialEmptyLine;
            potentialEmptyLine = true;
            s = null;
            break;

        case StreamTokenizer.TT_NUMBER:
            s = Integer.toString(new Double(tok.nval).intValue());
            potentialEmptyLine = false;
            break;

        case '\'':
            s = tok.sval;
            break;

        default:
            s = null;
        }

        //
        //  Assume that alternate words on the line are
        //  parameter and value, respectively.
        //
        if (s != null) {
            if (param == null) {
                param = s;
            } else {
                value = s;
                arglist.put(param, value);
                param = null;
            }
        }
    }

    //
    //  Now, we'll check the body.
    //

    if (potentialEmptyLine) {
        StringWriter out = new StringWriter();
        FileUtil.copyContents(in, out);

        String bodyContent = out.toString();

        if (bodyContent != null) {
            arglist.put(PARAM_BODY, bodyContent);
        }
    }

    return arglist;
}

---

From source file:gda.device.detector.mythen.data.MythenDataFileUtils.java

protected static double[][] getDataFromReaderUsingStreamTokenizer(Reader r, FileType type) throws IOException {
    try {/*ww  w.  j  a  v a2s.  c  o m*/
        List<double[]> data = new Vector<double[]>();
        StreamTokenizer st = new StreamTokenizer(r);
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            double angle = st.nval;
            st.nextToken();
            double count = st.nval;
            //st.nextToken();
            if (type == FileType.PROCESSED) {
                st.nextToken();
                double error = st.nval;
                data.add(new double[] { angle, count, error });
            } else if (type == FileType.PROCESSED_WITH_CHANNELS) {
                st.nextToken();
                double error = st.nval;
                double channel = st.nval;
                data.add(new double[] { angle, count, error, channel });
            } else {
                data.add(new double[] { angle, count });
            }
        }
        return data.toArray(new double[data.size()][]);
    } finally {
        try {
            r.close();
        } catch (IOException e) {
            // ignore
        }
    }
}

---

From source file:com.github.lindenb.jvarkit.tools.biostar.Biostar103303.java

private void readGTF(String uri, SAMSequenceDictionary dict) throws IOException {
    int count_exons = 0;
    final Set<String> unknown = new HashSet<String>();
    LOG.info("Reading " + uri);
    final Pattern tab = Pattern.compile("[\t]");
    final Map<String, GTFGene> transcript2gene = new HashMap<String, GTFGene>();
    LineIterator iter = IOUtils.openURIForLineIterator(uri);
    while (iter.hasNext()) {
        String line = iter.next();
        if (line.startsWith("#"))
            continue;
        String tokens[] = tab.split(line);
        if (tokens.length < 9)
            continue;
        if (!tokens[2].equals("exon"))
            continue;
        if (dict.getSequence(tokens[0]) == null) {
            if (!unknown.contains(tokens[0])) {
                LOG.warn("chromosome in " + line + " not in SAMSequenceDictionary ");
                unknown.add(tokens[0]);//from   w w  w. j  a v  a  2s. c  o m
            }
            continue;
        }
        String transcript_id = null, gene_id = null, gene_name = null, exon_id = null;
        StreamTokenizer st = new StreamTokenizer(new StringReader(tokens[8]));
        st.wordChars('_', '_');
        String key = null;
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            String s = null;
            switch (st.ttype) {
            case StreamTokenizer.TT_NUMBER:
                s = String.valueOf(st.nval);
                break;
            case '"':
            case '\'':
            case StreamTokenizer.TT_WORD:
                s = st.sval;
                break;
            case ';':
                break;
            default:
                break;
            }
            if (s == null)
                continue;
            if (key == null) {
                key = s;
            } else {
                if (key.equals("transcript_id")) {
                    transcript_id = s;
                } else if (key.equals("gene_id")) {
                    gene_id = s;
                } else if (key.equals("gene_name")) {
                    gene_name = s;
                } else if (key.equals("exon_id")) {
                    exon_id = s;
                }
                key = null;
            }
        }
        if (transcript_id == null || transcript_id.isEmpty())
            continue;
        GTFGene gene = transcript2gene.get(tokens[0] + " " + transcript_id);
        if (gene == null) {
            gene = new GTFGene();
            gene.transcript_id = transcript_id;
            gene.gene_id = gene_id;
            gene.gene_name = gene_name;
            gene.chrom = tokens[0];
            transcript2gene.put(tokens[0] + " " + transcript_id, gene);
        }
        GTFGene.Exon exon = gene.createExon(Integer.parseInt(tokens[3]), Integer.parseInt(tokens[4]));
        exon.exon_id = exon_id;
    }
    CloserUtil.close(iter);

    for (GTFGene g : transcript2gene.values()) {
        Collections.sort(g.exons, new Comparator<GTFGene.Exon>() {
            @Override
            public int compare(GTFGene.Exon o1, GTFGene.Exon o2) {
                return o1.start - o2.start;
            }
        });
        for (int i = 0; i < g.exons.size(); ++i) {

            GTFGene.Exon exon = g.exons.get(i);
            exon.index = i;

            if (i > 0) {
                GTFGene.Exon prev = g.exons.get(i - 1);
                if (prev.end >= exon.start) {
                    throw new IOException("exons " + (i) + " and " + (i + 1) + " overlap in " + g);
                }
            }

            Interval interval = new Interval(g.chrom, exon.start, exon.end);
            List<GTFGene.Exon> L = exonMap.get(interval);
            if (L == null) {
                L = new ArrayList<GTFGene.Exon>(1);
                exonMap.put(interval, L);
            }
            L.add(exon);
            ++count_exons;
        }
    }
    LOG.info("End Reading " + uri + " N=" + count_exons);
}

---

From source file:com.xpn.xwiki.util.Util.java

/**
 * Create a Map from a string holding a space separated list of key=value pairs. If keys or values must contain
 * spaces, they can be placed inside quotes, like <code>"this key"="a larger value"</code>. To use a quote as part
 * of a key/value, use <code>%_Q_%</code>.
 * //from ww  w . ja v a 2  s  .  c om
 * @param mapString The string that must be parsed.
 * @return A Map containing the keys and values. If a key is defined more than once, the last value is used.
 */
public static Hashtable<String, String> keyValueToHashtable(String mapString) throws IOException {
    Hashtable<String, String> result = new Hashtable<String, String>();
    StreamTokenizer st = new StreamTokenizer(new BufferedReader(new StringReader(mapString)));
    st.resetSyntax();
    st.quoteChar('"');
    st.wordChars('a', 'z');
    st.wordChars('A', 'Z');
    st.whitespaceChars(' ', ' ');
    st.whitespaceChars('=', '=');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        String key = st.sval;
        st.nextToken();
        String value = (st.sval != null) ? st.sval : "";
        result.put(key, restoreValue(value));
    }
    return result;
}

---

From source file:cross.io.xml.FragmentXMLSerializer.java

/**
 * @param name//from   w  w  w  . jav  a  2  s.  c om
 * @param dims
 * @param data
 * @return
 */
private Array handleData(final String name, final Dimension[] dims, final Element data, final Range[] ranges) {
    EvalTools.notNull(dims, this);
    final String dec = new String(Base64.decode(data.getText(), Base64.GZIP));
    final StreamTokenizer st = new StreamTokenizer(new StringReader(dec));
    final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
    final int[] shape = new int[dims.length];
    int d = 0;
    for (final Dimension dim : dims) {
        shape[d++] = dim.getLength();
    }
    Array a = null;
    IndexIterator idx = null;
    int tok = -1;
    // log.info("DataType of array: {}",
    // dt.getPrimitiveClassType().getName());
    Mode m = Mode.UNDEF;
    Object o = null;
    try {
        while ((tok = st.nextToken()) != StreamTokenizer.TT_EOL) {
            if (tok == StreamTokenizer.TT_WORD) {

                if (m == Mode.UNDEF) {
                    try {
                        o = nf.parse(st.sval);

                        if (o instanceof Double) {
                            m = Mode.DOUBLE;
                            a = Array.factory(DataType.DOUBLE, shape);
                        } else if (o instanceof Float) {
                            m = Mode.FLOAT;
                            a = Array.factory(DataType.FLOAT, shape);
                        } else if (o instanceof Long) {
                            m = Mode.LONG;
                            a = Array.factory(DataType.LONG, shape);
                        } else if (o instanceof Integer) {
                            m = Mode.INTEGER;
                            a = Array.factory(DataType.INT, shape);
                        } else if (o instanceof Byte) {
                            m = Mode.BYTE;
                            a = Array.factory(DataType.BYTE, shape);
                        } else if (o instanceof Short) {
                            m = Mode.SHORT;
                            a = Array.factory(DataType.SHORT, shape);
                        }
                    } catch (final ParseException pe) {
                        if (st.sval.equalsIgnoreCase("true") || st.sval.equalsIgnoreCase("false")) {
                            m = Mode.BOOLEAN;
                            a = Array.factory(DataType.BOOLEAN, shape);
                        } else {
                            m = Mode.STRING;
                            a = Array.factory(DataType.STRING, shape);
                        }
                    }
                } else {
                    if (idx == null) {
                        idx = a.getIndexIterator();
                    }
                    switch (m) {
                    case DOUBLE: {
                        idx.setDoubleNext((Double) o);
                        break;
                    }
                    case FLOAT: {
                        idx.setFloatNext((Float) o);
                        break;
                    }
                    case INTEGER: {
                        idx.setIntNext((Integer) o);
                        break;
                    }
                    case LONG: {
                        idx.setLongNext((Long) o);
                        break;
                    }
                    case BYTE: {
                        idx.setByteNext((Byte) o);
                        break;
                    }
                    case SHORT: {
                        idx.setShortNext((Short) o);
                        break;
                    }
                    case BOOLEAN: {
                        idx.setBooleanNext(Boolean.parseBoolean(st.sval));
                        break;
                    }
                    case STRING: {
                        idx.setObjectNext(st.sval);
                        break;
                    }
                    case OBJECT: {
                        throw new IllegalArgumentException("Could not handle type");
                    }
                    }
                }
            }
        }
    } catch (final IOException e) {
        log.warn("Could not read data for {}", name);
    }
    if (a != null && ranges != null && ranges.length != 0) {
        try {
            return a.section(Arrays.asList(ranges));
        } catch (InvalidRangeException ex) {
            log.warn("Invalid range while trying to subset array: ", ex);
        }
    }
    return a;
}

---

From source file:com.indeed.imhotep.index.builder.util.SmartArgs.java

/**
 * parse a String into an argument list, in the same way java parses
 * arguments passed to main()/*from www .j a v  a 2s  .c o  m*/
 */
public static String[] parseArgParams(String line) {
    StreamTokenizer tok = new StreamTokenizer(new StringReader(line));
    tok.resetSyntax();
    tok.wordChars('\u0000', '\uFFFF');
    tok.whitespaceChars(' ', ' ');
    tok.quoteChar('\"');
    ArrayList<String> output = new ArrayList<String>();
    try {
        while (tok.nextToken() != StreamTokenizer.TT_EOF) {
            output.add(tok.sval);
        }
    } catch (IOException e) {
    }
    return output.toArray(new String[output.size()]);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »