Lucene++ - a full-featured, c++ search engine
API Documentation
Main Page
Related Pages
Namespaces
Data Structures
Files
File List
Globals
All
Data Structures
Namespaces
Files
Functions
Variables
Typedefs
Enumerations
Enumerator
Macros
Pages
include
Tokenizer.h
Go to the documentation of this file.
1
2
// Copyright (c) 2009-2014 Alan Wright. All rights reserved.
3
// Distributable under the terms of either the Apache License (Version 2.0)
4
// or the GNU Lesser General Public License.
6
7
#ifndef TOKENIZER_H
8
#define TOKENIZER_H
9
10
#include "
TokenStream.h
"
11
12
namespace
Lucene {
13
20
class
LPPAPI
Tokenizer
:
public
TokenStream
{
21
protected
:
23
Tokenizer
();
24
26
Tokenizer
(
const
ReaderPtr
& input);
27
29
Tokenizer
(
const
AttributeFactoryPtr
& factory);
30
32
Tokenizer
(
const
AttributeFactoryPtr
& factory,
const
ReaderPtr
& input);
33
35
Tokenizer
(
const
AttributeSourcePtr
& source);
36
38
Tokenizer
(
const
AttributeSourcePtr
& source,
const
ReaderPtr
& input);
39
40
public
:
41
virtual
~
Tokenizer
();
42
43
LUCENE_CLASS
(
Tokenizer
);
44
45
protected
:
47
ReaderPtr
input;
48
CharStreamPtr
charStream
;
49
50
public
:
52
virtual
void
close();
53
59
virtual
int32_t correctOffset(int32_t currentOff);
60
61
using
TokenStream::reset
;
62
65
virtual
void
reset(
const
ReaderPtr
& input);
66
};
67
68
}
69
70
#endif
clucene.sourceforge.net