PythonTokenMarker.java :  » Game » aetheria » org » jedit » syntax » Java Open Source

package org.jedit.syntax;

/*
 * PythonTokenMarker.java - Python token marker
 * Copyright (C) 1999 Jonathan Revusky
 * Copyright (C) 1998, 1999 Slava Pestov
 *
 * You may use and modify this package for any purpose. Redistribution is
 * permitted, in both source and binary form, provided that this notice
 * remains intact in all source distributions of this package.
 */

import javax.swing.text.Segment;

/**
 * Python token marker.
 *
 * @author Jonathan Revusky
 * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
 */
public class PythonTokenMarker extends TokenMarker
{
  private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
  private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;

  public PythonTokenMarker()
  {
    this.keywords = getKeywords();
  }

  public byte markTokensImpl(byte token, Segment line, int lineIndex)
  {
    char[] array = line.array;
    int offset = line.offset;
    lastOffset = offset;
    lastKeyword = offset;
    int length = line.count + offset;
    boolean backslash = false;

loop:    for(int i = offset; i < length; i++)
    {
      int i1 = (i+1);

      char c = array[i];
      if(c == '\\')
      {
        backslash = !backslash;
        continue;
      }

      switch(token)
      {
      case Token.NULL:
        switch(c)
        {
        case '#':
          if(backslash)
            backslash = false;
          else
          {
            doKeyword(line,i,c);
            addToken(i - lastOffset,token);
            addToken(length - i,Token.COMMENT1);
            lastOffset = lastKeyword = length;
            break loop;
          }
          break;
        case '"':
          doKeyword(line,i,c);
          if(backslash)
            backslash = false;
          else
          {
            addToken(i - lastOffset,token);
            if(SyntaxUtilities.regionMatches(false,
              line,i1,"\"\""))
            {
              token = TRIPLEQUOTE1;
            }
            else
            {
              token = Token.LITERAL1;
            }
            lastOffset = lastKeyword = i;
          }
          break;
        case '\'':
          doKeyword(line,i,c);
          if(backslash)
            backslash = false;
          else
          {
            addToken(i - lastOffset,token);
            if(SyntaxUtilities.regionMatches(false,
              line,i1,"''"))
            {
              token = TRIPLEQUOTE2;
            }
            else
            {
              token = Token.LITERAL2;
            }
            lastOffset = lastKeyword = i;
          }
          break;
        default:
          backslash = false;
          if(!Character.isLetterOrDigit(c)
            && c != '_')
            doKeyword(line,i,c);
          break;
        }
        break;
      case Token.LITERAL1:
        if(backslash)
          backslash = false;
        else if(c == '"')
        {
          addToken(i1 - lastOffset,token);
          token = Token.NULL;
          lastOffset = lastKeyword = i1;
        }
        break;
      case Token.LITERAL2:
        if(backslash)
          backslash = false;
        else if(c == '\'')
        {
          addToken(i1 - lastOffset,Token.LITERAL1);
          token = Token.NULL;
          lastOffset = lastKeyword = i1;
        }
        break;
      case TRIPLEQUOTE1:
        if(backslash)
          backslash = false;
        else if(SyntaxUtilities.regionMatches(false,
          line,i,"\"\"\""))
        {
          addToken((i+=4) - lastOffset,
            Token.LITERAL1);
          token = Token.NULL;
          lastOffset = lastKeyword = i;
        }
        break;
      case TRIPLEQUOTE2:
        if(backslash)
          backslash = false;
        else if(SyntaxUtilities.regionMatches(false,
          line,i,"'''"))
        {
          addToken((i+=4) - lastOffset,
            Token.LITERAL1);
          token = Token.NULL;
          lastOffset = lastKeyword = i;
        }
        break;
      default:
        throw new InternalError("Invalid state: "
          + token);
      }
    }

    switch(token)
    {
      case TRIPLEQUOTE1:
      case TRIPLEQUOTE2:
        addToken(length - lastOffset,Token.LITERAL1);
        break;
      case Token.NULL:
        doKeyword(line,length,'\0');
      default:
        addToken(length - lastOffset,token);
        break;
    }

    return token;
  }

  public static KeywordMap getKeywords()
  {
    if (pyKeywords == null)
    {
      pyKeywords = new KeywordMap(false);
      pyKeywords.add("and",Token.KEYWORD3);
      pyKeywords.add("not",Token.KEYWORD3);
      pyKeywords.add("or",Token.KEYWORD3);
      pyKeywords.add("if",Token.KEYWORD1);
      pyKeywords.add("for",Token.KEYWORD1);
      pyKeywords.add("assert",Token.KEYWORD1);
      pyKeywords.add("break",Token.KEYWORD1);
      pyKeywords.add("continue",Token.KEYWORD1);
      pyKeywords.add("elif",Token.KEYWORD1);
      pyKeywords.add("else",Token.KEYWORD1);
      pyKeywords.add("except",Token.KEYWORD1);
      pyKeywords.add("exec",Token.KEYWORD1);
      pyKeywords.add("finally",Token.KEYWORD1);
      pyKeywords.add("raise",Token.KEYWORD1);
      pyKeywords.add("return",Token.KEYWORD1);
      pyKeywords.add("try",Token.KEYWORD1);
      pyKeywords.add("while",Token.KEYWORD1);
      pyKeywords.add("def",Token.KEYWORD2);
      pyKeywords.add("class",Token.KEYWORD2);
      pyKeywords.add("del",Token.KEYWORD2);
      pyKeywords.add("from",Token.KEYWORD2);
      pyKeywords.add("global",Token.KEYWORD2);
      pyKeywords.add("import",Token.KEYWORD2);
      pyKeywords.add("in",Token.KEYWORD2);
      pyKeywords.add("is",Token.KEYWORD2);
      pyKeywords.add("lambda",Token.KEYWORD2);
      pyKeywords.add("pass",Token.KEYWORD2);
      pyKeywords.add("print",Token.KEYWORD2);
    }
    return pyKeywords;
  }

  // private members
  private static KeywordMap pyKeywords;

  private KeywordMap keywords;
  private int lastOffset;
  private int lastKeyword;

  private boolean doKeyword(Segment line, int i, char c)
  {
    int i1 = i+1;

    int len = i - lastKeyword;
    byte id = keywords.lookup(line,lastKeyword,len);
    if(id != Token.NULL)
    {
      if(lastKeyword != lastOffset)
        addToken(lastKeyword - lastOffset,Token.NULL);
      addToken(len,id);
      lastOffset = i;
    }
    lastKeyword = i1;
    return false;
  }
}