;; LEX-STRING separates a STRING into tokens at WHITESPACE-CHARS (by default, space
;; tab, newpage, backspace, return, linefeed and newline).

(defconstant +whitespace-characters+
  '(#\space #\tab #\page #\newline #\backspace #\return #\linefeed #\no-break_space)
  "The whitespace characters")

(defun lex-string (string &optional (whitespace-chars +whitespace-characters+))
  "Separates a string at whitespace and returns a list of strings"
  (assert (and (stringp string) (every #'characterp whitespace-chars)))
  (flet ((whitespace-char? (char) (find char whitespace-chars :test #'char=)))
    (let ((tokens nil)
	  token-end)
      (do ((token-start
	    (position-if-not #'whitespace-char? string) 
	    (when token-end
	      (position-if-not #'whitespace-char? string :start (1+ token-end)))))
	  ((null token-start) (nreverse tokens))
	(setq token-end
	      (when token-start
		(position-if #'whitespace-char? string :start token-start)))
	(push (subseq string token-start token-end) tokens)))))