diff options
Diffstat (limited to 'src/main/java/org/pdfbox/searchengine/lucene/IndexFiles.java')
| -rw-r--r-- | src/main/java/org/pdfbox/searchengine/lucene/IndexFiles.java | 308 |
1 files changed, 0 insertions, 308 deletions
diff --git a/src/main/java/org/pdfbox/searchengine/lucene/IndexFiles.java b/src/main/java/org/pdfbox/searchengine/lucene/IndexFiles.java deleted file mode 100644 index dd36dd9..0000000 --- a/src/main/java/org/pdfbox/searchengine/lucene/IndexFiles.java +++ /dev/null @@ -1,308 +0,0 @@ -package org.pdfbox.searchengine.lucene;
-
-/*
- * This source was originally written as an example for the lucene project.
- * It has been modified to use PDFBox as a lucene document creator.
- * -Ben Litchfield
- *
- *====================================================================
- * The Apache Software License, Version 1.1
- *
- * Copyright (c) 2001 The Apache Software Foundation. All rights
- * reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * 3. The end-user documentation included with the redistribution,
- * if any, must include the following acknowledgment:
- * "This product includes software developed by the
- * Apache Software Foundation (http://www.apache.org/)."
- * Alternately, this acknowledgment may appear in the software itself,
- * if and wherever such third-party acknowledgments normally appear.
- *
- * 4. The names "Apache" and "Apache Software Foundation" and
- * "Apache Lucene" must not be used to endorse or promote products
- * derived from this software without prior written permission. For
- * written permission, please contact apache@apache.org.
- *
- * 5. Products derived from this software may not be called "Apache",
- * "Apache Lucene", nor may "Apache" appear in their name, without
- * prior written permission of the Apache Software Foundation.
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
- * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- * ====================================================================
- *
- * This software consists of voluntary contributions made by many
- * individuals on behalf of the Apache Software Foundation. For more
- * information on the Apache Software Foundation, please see
- * <http://www.apache.org/>.
- */
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-
-import org.apache.lucene.demo.HTMLDocument;
-
-import org.apache.lucene.document.Document;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
-
-import java.util.Arrays;
-
-
-import java.io.File;
-import java.io.IOException;
-
-import java.util.Date;
-
-
-/**
- * This is a class that will index some files on a local filesystem. This code
- * was modified from a demo that comes with the lucene search engine.
- *
- * @author Lucene team
- * @author Ben Litchfield (ben@csh.rit.edu)
- *
- * @version $Revision: 1.6 $
- */
-public class IndexFiles
-{
- private boolean deleting = false; // true during deletion pass
- private IndexReader reader; // existing index
- private IndexWriter writer; // new index being built
- private TermEnum uidIter; // document id iterator
-
- /**
- * This is the main entry point for the indexer.
- *
- * @param argv The command line arguments.
- */
- public static void main(String[] argv)
- {
-
- String index = "index";
- boolean create = false;
- File root = null;
-
- String usage = "org.pdfbox.searchengine.lucene.IndexFiles [-create] [-index <index>] <root_directory>";
-
- if (argv.length == 0)
- {
- System.err.println("Usage: " + usage);
- return;
- }
-
- for (int i = 0; i < argv.length; i++)
- {
- if (argv[i].equals("-index"))
- { // parse -index option
- index = argv[++i];
- }
- else if (argv[i].equals("-create"))
- { // parse -create option
- create = true;
- }
- else if (i != argv.length-1)
- {
- System.err.println("Usage: " + usage);
- return;
- }
- else
- {
- System.out.println( "root=" +argv[i] );
- root = new File(argv[i]);
- }
- }
- IndexFiles indexer = new IndexFiles();
- indexer.index( root, create, index );
- }
-
- /**
- * This will index a directory.
- *
- * @param root The root directory to start indexing.
- * @param create Should we create a new index?
- * @param index The name of the index.
- */
- public void index( File root, boolean create, String index )
- {
-
- try
- {
- Date start = new Date();
-
- writer = new IndexWriter(index, new StandardAnalyzer(), create);
- writer.maxFieldLength = 1000000;
-
- if (!create)
- { // delete stale docs
- deleting = true;
- indexDocs(root, index, create);
- }
-
- indexDocs(root, index, create); // add new docs
-
- System.out.println("Optimizing index...");
- writer.optimize();
- writer.close();
-
- Date end = new Date();
-
- System.out.print(end.getTime() - start.getTime());
- System.out.println(" total milliseconds");
-
- }
- catch( Exception e )
- {
- e.printStackTrace();
- }
- }
-
- /**
- * Walk directory hierarchy in uid order, while keeping uid iterator from
- * existing index in sync. Mismatches indicate one of: (a) old documents to
- * be deleted; (b) unchanged documents, to be left alone; or (c) new
- * documents, to be indexed.
- *
- * @param file The directory to index.
- * @param index The index to add the file to.
- * @param create A flag telling if we should create the index.
- *
- * @throws Exception If there is any error indexing the directory.
- */
- private void indexDocs(File file, String index, boolean create) throws Exception
- {
- if (!create)
- { // incrementally update
-
- reader = IndexReader.open(index); // open existing index
- uidIter = reader.terms(new Term("uid", "")); // init uid iterator
-
- indexDocs(file);
-
- if (deleting)
- { // delete rest of stale docs
- while (uidIter.term() != null && uidIter.term().field().equals( "uid" ) )
- {
- System.out.println("deleting " +
- HTMLDocument.uid2url(uidIter.term().text()));
- reader.delete(uidIter.term());
- uidIter.next();
- }
- deleting = false;
- }
-
- uidIter.close(); // close uid iterator
- reader.close(); // close existing index
-
- }
- else
- {
- indexDocs(file);
- }
- }
-
-
- private void indexDocs(File file) throws Exception
- {
- if (file.isDirectory())
- { // if a directory
- String[] files = file.list(); // list its files
- Arrays.sort(files); // sort the files
- for (int i = 0; i < files.length; i++) // recursively index them
- {
- indexDocs(new File(file, files[i]));
- }
- }
- else
- {
- if (uidIter != null)
- {
- String uid = HTMLDocument.uid(file); // construct uid for doc
-
- while( uidIter.term() != null &&
- uidIter.term().field().equals( "uid" ) &&
- uidIter.term().text().compareTo(uid) < 0)
- {
- if (deleting)
- { // delete stale docs
- System.out.println("deleting " +
- HTMLDocument.uid2url(uidIter.term().text()));
- reader.delete(uidIter.term());
- }
- uidIter.next();
- }
- if( uidIter.term() != null &&
- uidIter.term().field().equals( "uid" ) &&
- uidIter.term().text().compareTo(uid) == 0)
- {
- System.out.println( "Next uid=" +uidIter );
- uidIter.next(); // keep matching docs
- }
- }
- else
- {
- try
- {
- addDocument( file );
- }
- catch( IOException e )
- {
- //catch exception and move onto the next document
- System.out.println( e.getMessage() );
- }
- }
- }
- }
-
- private void addDocument( File file ) throws IOException, InterruptedException
- {
- String path = file.getName().toUpperCase();
- Document doc = null;
- //Gee, this would be a great place for a command pattern
- if( path.endsWith(".HTML") || // index .html files
- path.endsWith(".HTM") || // index .htm files
- path.endsWith(".TXT"))
- {
- System.out.println( "Indexing Text document: " + file );
- doc = HTMLDocument.Document(file);
- }
- else if( path.endsWith( ".PDF" ) )
- {
- System.out.println( "Indexing PDF document: " + file );
- doc = LucenePDFDocument.getDocument( file );
- }
- else
- {
- System.out.println( "Skipping " + file );
- }
-
- if( doc != null )
- {
- writer.addDocument(doc);
- }
- }
-}
\ No newline at end of file |