The string tokenizer class allows an application to break a string into tokens.
@author Azeem Michael
@example The following is one example of the use of the tokenizer. The code:
<?php
$str = "this is:@tn a test!";
$delim = " !@:'tn"; // remove these chars
$st = new StringTokenizer($str,$delim);
echo 'Total tokens: '.$st->countTokens().' ';
while ($st->hasMoreTokens()) {
echo $st->nextToken() . ' ';
}
prints the following output:
Total tokens: 4
this
is
a
test
?>
*/
class StringTokenizer {
/* @var string /
private $string;
/* @var string /
private $token;
/* @var string /
private $delim;
/**
Constructs a string tokenizer for the specified string.
@param string $str String to tokenize
@param string $delim The set of delimiters (the characters that separate tokens)
specified at creation time,default to " nrt"
*/
public function __construct($str,$delim=" nrt") {
$this->string = $str;
$this->delim = $delim;
$this->token = strtok($str,$delim);
}
/**
Destructor to prevent memory leaks
*/
public function __destruct() {
unset($this);
}
/**
Calculates the number of times that this tokenizer's nextToken method can
be called before it generates an exception
@return int - number of tokens
*/
public function countTokens() {
$counter = 0;
while($this->hasMoreTokens()) {
$counter++;
$this->nextToken();
}
$this->token = strtok($this->string,$this->delim);
return $counter;
}
/**
Tests if there are more tokens available from this tokenizer's string. It
does not move the internal pointer in any way. To move the internal pointer
to the next element call nextToken()
@return boolean - true if has more tokens,false otherwise
*/
public function hasMoreTokens() {
return ($this->token !== false);
}
/**
Returns the next token from this string tokenizer and advances the internal
pointer by one.
@return string - next element in the tokenized string
*/
public function nextToken() {
$hold = $this->token; //hold current pointer value
$this->token = strtok($this->delim); //increment pointer
return $hold; //return current pointer value
}
}
?>
