F8/src/MOS/6502/assembler/tokenizer.js

function GetTextStream(input){
  var pos = 0;
  var line = 0;
  var col = 0;

  function peek(){
    return input.getChar(pos);
  }

  function next(){
    let c = input.charAt(pos);
    pos += 1;
    if (c === "\n"){
      line += 1;
      col = 0;
    } else {col += 1;}
    return c;
  }

  function eof(){
    return (input.charAt(pos) === "");
  }

  function die(msg){
    throw new Error(msg + " Line: " + line + ", Col: " + col);
  }

  return {
    peek: peek,
    get: next,
    eof: eof
  };
}

// ----------------------------------------------
// VALIDATORS
// ----------------------------------------------

function isWhiteSpace(c){
  return ("\t\n ".indexOf(c) >= 0);
}

function isStringStart(c){
  return (c === "\"" || c === "'");
}

function isNumType(c){
  return ("$%".indexOf(c) >= 0 || isDigit(c));
}

function isDigit(c){
  return /[0-9]/i.test(c);
}

function isHex(c){
  return /[0-9a-fA-F]/i.test(c);
}

function isBinary(c){
  return ("01".indexOf(c) >= 0);
}

function isLabelStart(c){
  return /[a-fA-F_]/i.test(c);
}

function isLabel(c){
  return (isLabelStart(c) || isDigit(c));
}

function isPunctuation(c){
  return (",()".indexOf(c) >= 0);
}

function isOp(c){
  return ("=+-/*".indexOf(c) >= 0);
}


// ----------------------------------------------
// Read Operations
// ----------------------------------------------

function skipComment(stream){
  readWhile(stream, (c)=>{return c != "\n";});
  stream.next();
}

function readHex(stream){
  stream.next();
  var str = readWhile(stream, isHex);
  return {type:'number', val: parseInt(str, 16)};
}

function readBinary(stream){
  stream.next();
  var str = readWhile(stream, isBinary);
  return {type:'number', val: parseInt(str, 2)};
}

function readNumber(stream){
  let c = stream.peek();
  if (c === "$")
    return readHex(stream);
  if (c === "%")
    return readBinary(stream);

  var dot = false;
  var str = readWhile(stream, (c)=>{
    if (c === "."){
      if (dot){return false;}
      dot = true;
      return true;
    }
    return isDigit(c);
  });
  return {type:'number', val:parseFloat(str)};
}


function readString(stream, end){
  var str = "";
  var escaped = false;
  stream.next();
  while (!stream.eof()){
    let c = stream.next();
    if (escaped){
      str += c;
      escaped = false;
    } else if (c === "\\"){
      escaped = true;
    } else if (c === end){
      break;
    } else {
      str += c;
    }
  }
  return {type: "string", val: str};
}

function readLabel(stream){
  str = readWhile(stream, isLabel);
  return {type:"label", val: str};
}

function readWhile(stream, validator){
  var str = "";
  while (!stream.eof() && validator(stream.peek()))
    str += stream.next();
  return str;
}

function nextToken(stream){
  readWhile(stream, isWhiteSpace);
  if (stream.eof()){return null;}
  let c = stream.peek();
  if (c === ";"){
    skipComment(stream);
    return nextToken(stream);
  } else if (isStringStart(c)){
    return readString(stream, c);
  } else if (isNumType(c)){
    return readNumber(stream);
  } else if (isLabelStart(c)){
    return readLabel(stream);
  } else if (isPunctuation(c)){
    return {type:"punctuation", val:stream.next()};
  } else if (isMathOp(c)){
    return {type:"op", val:stream.next()};
  }
  stream.die("Unable to process character '" + c + "'.");
}


function tokenize(input){
  var stream = GetTextStream(input);
  var tokens = [];
  var t = nextToken(stream);
  while (t !== null){
    tokens.push(t);
    t = nextToken(stream);
  }
  return tokens;
}


module.exports = tokenize;