// 안녕하세요  김영대(http://www.howto.pe.kr) 입니다  

unit Unit1;

interface

uses
  Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
  Dialogs, StdCtrls;

type
  TForm1 = class(TForm)
    Edit1: TEdit;
    Memo1: TMemo;
    Button1: TButton;
    procedure Button1Click(Sender: TObject);
    procedure FormCreate(Sender: TObject);
  private
    { Private declarations }
  public
    { Public declarations }
  end;

type
  TToken = record
    word: String; // word(token)
    start: integer; // word의 시작 위치
    len: integer; // word의 길이
  end;

const
  Delimiters = '?=@+;!.,?_/~&[]<>#`{}:$%^&()*;" '+#9#10#13#44#46#59;

var
  Form1: TForm1;

implementation
{$R *.dfm}

function GetTToken(const text: string; var Token: TToken; var index: integer): Boolean;
  function isDelimiter(const c: char): boolean;
  begin
    Result := Pos(c, delimiters) > 0;
  end;
var
  TextLength: integer;
begin
  with Token do
  begin
    TextLength := length(text);

    // 연속하는 구분자들을 skip 한다
    start := index;
    while isDelimiter(text[start]) and (start < TextLength) do
      inc(start);

    // 더이상 token 없음
    if start > TextLength then
    begin
      word := '';
      Result := false;
      exit
    end;

    // token을 구한다      
    len := start;
    while not isDelimiter(text[len]) do
      inc(len);
    len := len-start; // 길이
    word := copy(text, start, len); // token
    index := start+len; // 시작위치

    Result := word <> '';
  end;
end;

procedure TForm1.Button1Click(Sender: TObject);
var
  index: integer;
  token: TToken;
begin
   index := 1;
   while GetTToken(Edit1.Text, token, index) do
     Memo1.Lines.add('토큰:'+token.word+
                     ',  시작위치:'+inttostr(token.start)+
                     ',  길이:'+inttostr(token.len));
end;

procedure TForm1.FormCreate(Sender: TObject);
begin
  Edit1.Text := ',김영대,우하하;3455;"howto"';
end;

end.