method tokenize

Source

List<Token> tokenize(String inputString) {
  List<Token> tempTokenStream = new List<Token>();

  String clearedString = inputString.replaceAll(" ", "").trim();

  RuneIterator iter = clearedString.runes.iterator;
  int siInt;

  while (iter.moveNext()) {
    String si = iter.currentAsString;

    /*
     * Check if the current Character is a keyword. If it is a keyword, check if the intBuffer is not empty and add
     * a Value Token for the intBuffer and the corresponding Token for the keyword.
     */
    if(keywords.containsKey(si)) {
      // check and or do intBuffer and varBuffer
      if(intBuffer.length > 0) {
        _doIntBuffer(tempTokenStream);
      }
      if(varBuffer.length > 0) {
        _doVarBuffer(tempTokenStream);
      }
      // MH - Bit of a hack here to handle exponentials of the form e^x rather than e(x)
      if (keywords[si] == TokenType.POW && tempTokenStream.last.type == TokenType.EFUNC) {
        // Clear varBuffer since we have nothing to add to the stream as EFUNC is already in it
        //_doVarBuffer(tempTokenStream);
        varBuffer = "";
      } else {
        // Normal behaviour
        tempTokenStream.add(new Token(si, keywords[si]));
      }
    } else {
      // Check if the current string is a Number. If it's the case add the string to the intBuffer.
      StringBuffer sb = new StringBuffer(intBuffer);
      try {
        siInt = int.parse(si);
        // The current string is a number and it is added to the intBuffer.
        sb.write(si);
        intBuffer = sb.toString();
        if(varBuffer.length > 0) {
          _doVarBuffer(tempTokenStream);
        }
      } on FormatException {
        // Check if the current string is part of a floating point input
        if(si=="."){
          sb.write(si);
          intBuffer = sb.toString();
          continue;
        }

        // The current string is not a number and not a simple keyword, so it has to be a variable or function.
        sb = new StringBuffer(varBuffer);
        if(intBuffer.length > 0) {
          /*
           * The intBuffer contains a string and the current string is a
           * variable or part of a complex keyword, so the value is added
           * to the token stream and the current string is added to the
           * var buffer.
           */
          _doIntBuffer(tempTokenStream);
          sb.write(si);
          varBuffer = sb.toString();
        } else {
          // intBuffer contains no string and the current string is a variable, so both Tokens are added to the tokenStream.
          sb.write(si);
          varBuffer = sb.toString();
        }
      }
    }
  }

  if(intBuffer.length > 0) {
    // There are no more symbols in the input string but there is still an int in the intBuffer
    _doIntBuffer(tempTokenStream);
  }
  if(varBuffer.length > 0) {
    // There are no more symbols in the input string but there is still a variable or keyword in the varBuffer
    _doVarBuffer(tempTokenStream);
  }
  return tempTokenStream;
}