F8/src/MOS/6502/assembler/tokenizer.js

function GetTextStream(input){
  var pos = 0;
  var line = 0;
  var col = 0;

  function peek(){
    return input.charAt(pos);
  }

  function next(){
    let c = input.charAt(pos);
    pos += 1;
    if (c === "\n"){
      line += 1;
      col = 0;
    } else {col += 1;}
    return c;
  }

  function eof(){
    return (input.charAt(pos) === "");
  }

  function getLine(){return line;}
  function getCol(){return col;}

  function die(msg){
    throw new Error(msg + " Line: " + line + ", Col: " + col);
  }

  return {
    peek: peek,
    next: next,
    eof: eof,
    line: getLine,
    col: getCol,
    die: die
  };
}


// ----------------------------------------------
// VALIDATORS
// ----------------------------------------------

function isWhiteSpace(c){
  return ("\t\n ".indexOf(c) >= 0);
}

function isStringStart(c){
  return (c === "\"" || c === "'");
}

function isNumType(c){
  return ("$%".indexOf(c) >= 0 || isDigit(c));
}

function isDigit(c){
  return /[0-9]/i.test(c);
}

function isHex(c){
  return /[0-9a-fA-F]/i.test(c);
}

function isBinary(c){
  return ("01".indexOf(c) >= 0);
}

function isLabelStart(c){
  return /[a-zA-Z_]/i.test(c);
}

function isLabel(c){
  return (isLabelStart(c) || isDigit(c));
}

function isPunctuation(c){
  return (",()".indexOf(c) >= 0);
}

function isOp(c){
  return ("=+-/*#".indexOf(c) >= 0);
}


// -----------------------------------------------------------
// TOKENIZER CLASS
// -----------------------------------------------------------

class Tokenizer{

  constructor(input){
    this.__stream = GetTextStream(input);
  }

  genTokenObject(type, val){
    return {
      type: type,
      val: val,
      line: this.__stream.line(),
      col: this.__stream.col()
    }
  }

// ----------------------------------------------
// Read Operations
// ----------------------------------------------

  skipComment(){
    this.readWhile((c)=>{return c != "\n";});
    this.__stream.next();
  }

  readHex(){
    this.__stream.next();
    var str = this.readWhile(isHex);
    return this.genTokenObject('number', parseInt(str, 16));
  }

  readBinary(){
    this.__stream.next();
    var str = this.readWhile(isBinary);
    return this.genTokenObject('number', parseInt(str, 2));
  }

  readNumber(){
    let c = this.__stream.peek();
    if (c === "$")
      return this.readHex();
    if (c === "%")
      return this.readBinary();

    var str = this.readWhile(isDigit);
    return this.genTokenObject('number', parseInt(str));
  }


  readString(end){
    var str = "";
    var escaped = false;
    this.__stream.next();
    while (!this.__stream.eof()){
      let c = this.__stream.next();
      if (escaped){
        str += c;
        escaped = false;
      } else if (c === "\\"){
        escaped = true;
      } else if (c === end){
        break;
      } else {
        str += c;
      }
    }
    return this.genTokenObject('string', str);
  }

  readDirective(){
    var str = this.__stream.next() + this.readWhile(isLabel);
    return this.genTokenObject('directive', str);
  }

  readLabel(){
    var str = this.readWhile(isLabel);
    return this.genTokenObject('label', str);
  }

  readWhile(validator){
    var str = "";
    while (!this.__stream.eof() && validator(this.__stream.peek()))
      str += this.__stream.next(); 
    return str;
  }

  nextToken(){
    this.readWhile(isWhiteSpace);
    if (this.__stream.eof()){return null;}
    let c = this.__stream.peek();
    if (c === ";"){
      this.skipComment();
      return this.nextToken();
    } else if (isStringStart(c)){
      return this.readString(c);
    } else if (isNumType(c)){
      return this.readNumber();
    } else if (isLabelStart(c)){
      return this.readLabel();
    } else if (c === "."){
      return this.readDirective();
    } else if (isPunctuation(c)){
      return this.genTokenObject('punc', this.__stream.next());
    } else if (isOp(c)){
      return this.genTokenObject('op', this.__stream.next());
    }
    this.__stream.die("Unable to process character '" + c + "'.");
  }

}

function tokenize(input){
  var tokenizer = new Tokenizer(input);
  var tokens = [];
  var t = tokenizer.nextToken();
  while (t !== null){
    tokens.push(t);
    t = tokenizer.nextToken();
  }
  return tokens;
}



module.exports = Object.freeze({
  Tokenizer:Tokenizer,
  tokenize:tokenize
});