#include "token.hh"
#include "language.hh"

#define increment__ \
    { if (endf_(next)) {cur_token.end = next; return *this;} \
    prev = cur; cur = next; ++next; }

template <typename Itr, typename EndF>
tok<Itr,EndF>& tok<Itr,EndF>::operator++() {
  cur_token.begin = cur_token.end;
  real_itr prev,cur,next;
  
  // figure out what the current token is
  cur = cur_token.begin;
  if (endf_(cur)) {cur_token.end = cur; return *this;}
  cur_token.is_word = lang->is_alpha(*cur);
  next = cur; ++next;
  if (lang->special(*cur).begin && !endf_(next) && lang->is_alpha(*next))
    cur_token.is_word = true;
  
  increment__
  
  if (cur_token.is_word) {
    
    while (lang->is_alpha(*cur) || 
	   (lang->special(*cur).middle && 
	    lang->is_alpha(*prev) &&
	    !endf_(next) && lang->is_alpha(*next) ))
      increment__
    
    // cur is nonalpha
    if (lang->special(*cur).end) cur_token.end = next;
    else                         cur_token.end = cur;
    
  } else {
    
    while (!lang->is_alpha(*cur))
      increment__
    
    // cur is alpha
    if (lang->special(*prev).begin) cur_token.end = prev;
    else                            cur_token.end = cur;
    
  }
  return *this;
}

#undef increment__

