java lucene中文分词

Lucene是一个全文检索引擎工具包，貌似挺好用。某些时候我们需要在数据库全表扫描筛选数据时，如果数据量庞大，往往要等待很多时间，这对用户来说是很不友好的。那么这时Lucene就可以派上用场。

Lucene首先将预检索资源封装成document对象，然后根据你自定义的字段建立索引，这其实和数据库的行为类似。

1.文档字符化

2. 对查询关键字分词

3..建立索引

4. 搜索

程序运行环境JDK1.6，主要使用的几个jar包：

具体实例代码如下：

CreateIndex.java (根据指定文件创建索引库)

package lucene;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;

import jeasy.analysis.MMAnalyzer;

public class CreateIndex {
	/**
	 * 创建索引库
	 */
	public static void initIndex() {
		File fileDir = new File(Constants.indexPath);
		IndexWriter indexWriter = null;
		try {
			Analyzer analyzer = new MMAnalyzer(); // 极易中文分词器
			//这里的true表示是否重新创建
			indexWriter = new IndexWriter(fileDir, analyzer,true , MaxFieldLength.LIMITED);
			List<File> fileList = getFiles();
			int count = fileList.size();
			for (int i = 0; i < count; i++) {
				Document document = null;
				document = fileToText(fileList.get(i));
				indexWriter.addDocument(document);
			}
			System.out.println("创建索引库成功！");
		} catch (Exception e) {
			System.err.println("不能正确创建索引库");
			fileDir.deleteOnExit(); // 如果创建索引库失败，则删除已经创建的索引目录，下次重新创建
		} finally {
			try {
				if (indexWriter != null)
					indexWriter.close();
			} catch (Exception e) {
				System.err.println("不能关闭indexWriter");
			}
		}
	}
	/**
	 * 将文件内容封装成Document
	 * @param file
	 * @return
	 * @throws IOException
	 */
	private static Document fileToText(File file) throws IOException {
		Document document = new Document();
		document.add(new Field("path",file.getAbsolutePath(),Store.YES,Index.NOT_ANALYZED));
		document.add(new Field("colContent",getContent(file),Store.YES,Index.ANALYZED));
		return document;
	}
	
	/**
	 * 文档字符化
	 * @param file
	 * @return
	 */
	public static String getContent(File file) {
		byte[]buffer = new byte[1024];
		StringBuffer sb = new StringBuffer();
		try {
			InputStream in = new FileInputStream(file);
			while(in.read(buffer) > 0) {
				sb.append(new String(buffer));
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		return sb.toString();
	}
	public static List<File> getFiles() {
		File file = new File(Constants.filePath);
		return Arrays.asList(file.listFiles());
	}
	
	public static void main(String[] args) {
		initIndex();
	}
}

IndexSearch.java (根据关键词进行搜索)

package lucene;

import java.io.IOException;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;

import jeasy.analysis.MMAnalyzer;

public class IndexSearch {
	private static Analyzer analyzer = new MMAnalyzer(); // 极易中文分词器
	public static void main(String[] args) throws Exception {
		IndexSearch search = new IndexSearch();
		search.search("姓名");
	}
	public void search(String keyWord) throws Exception {
		IndexSearcher indexSearcher = null;
		QueryParser queryParser = new MultiFieldQueryParser(new String[]{"colContent"}, analyzer);
		Query query = null;
		try {
			// 将关键字转换成索引库可以识别的Query对象
			query = queryParser.parse(keyWord);
		} catch (ParseException e) {
			System.err.println("关键词解析失败!");
		}
		indexSearcher = new IndexSearcher(Constants.indexPath);
		
		if (indexSearcher != null) {
			TopDocs topDocs;
			try {
				topDocs = indexSearcher.search(query, null, 10);
				ScoreDoc[] scoreDocs = topDocs.scoreDocs;
				Document document = null;
				System.out.println("共找到匹配文件: " + scoreDocs.length + "个");
				for (int i = 0; i < scoreDocs.length; i++) {
					ScoreDoc scorDoc = scoreDocs[i];
					int doc = scorDoc.doc;
					document = indexSearcher.doc(doc);
					System.out.println("文件路径:"+document.getField("path").stringValue());
					System.out.println("内容:"+document.getField("colContent").stringValue());
					System.out.println("++++++++++++++++++++++++++++++");
				}
			} catch (IOException e) {
				System.err.println("索引库查询失败");
				e.printStackTrace();
			} finally {
				try {
					if (indexSearcher != null) {
						indexSearcher.close();
					}
				} catch (Exception e) {
					System.err.println("不能关闭indexSearcher连接");
				}
			}
		}
	}
}

程序执行结果：

注意 ： 我这里是分两个类分别运行的，首先运行CreateIndex类建立索引库，然后，执行IndexSearch进行搜索。。更多内容在慢慢研究中