Yonik: can you clarify some of the items I labeled "TODO" ... I'm a little lost in understanding the distinction between internal, external, stored, indexed, and readable values.
: Date: Tue, 06 Jun 2006 07:53:41 -0000 : From: [EMAIL PROTECTED] : Reply-To: solr-dev@lucene.apache.org : To: solr-commits@lucene.apache.org : Subject: svn commit: r412029 - in /incubator/solr/trunk: build.xml : src/java/org/apache/solr/schema/FieldType.java : src/java/org/apache/solr/schema/IndexSchema.java : src/java/org/apache/solr/schema/SchemaField.java : : Author: hossman : Date: Tue Jun 6 00:53:40 2006 : New Revision: 412029 : : URL: http://svn.apache.org/viewvc?rev=412029&view=rev : Log: : more improvements to javadocs : : Modified: : incubator/solr/trunk/build.xml : incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java : incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java : incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java : : Modified: incubator/solr/trunk/build.xml : URL: http://svn.apache.org/viewvc/incubator/solr/trunk/build.xml?rev=412029&r1=412028&r2=412029&view=diff : ============================================================================== : --- incubator/solr/trunk/build.xml (original) : +++ incubator/solr/trunk/build.xml Tue Jun 6 00:53:40 2006 : @@ -92,7 +92,24 @@ : : : <target name="javadoc" depends="compile"> : + : <mkdir dir="${build.javadoc}"/> : + : + <!-- we do this to make sure whatever classes where in ant's : + classpath at runtime are in the classpath used by javadoc : + (ie: junit.jar) : + --> : + <path id="javadoc.classpath"> : + <path refid="compile.classpath"/> : + <!-- aparently ant.library.dir isn't allways set right? --> : + <fileset dir="${ant.home}/lib"> : + <include name="**/*.jar"/> : + </fileset> : + <fileset dir="${ant.library.dir}"> : + <include name="**/*.jar"/> : + </fileset> : + </path> : + : <javadoc : destdir="${build.javadoc}" : author="true" : @@ -109,7 +126,7 @@ : <link href="${javadoc.link.java}"/> : <link href="${javadoc.link.junit}"/> : <link href="${javadoc.link.lucene}"/> : - <classpath refid="compile.classpath"/> : + <classpath refid="javadoc.classpath"/> : </javadoc> : </target> : : : Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java : URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java?rev=412029&r1=412028&r2=412029&view=diff : ============================================================================== : --- incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java (original) : +++ incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java Tue Jun 6 00:53:40 2006 : @@ -44,12 +44,17 @@ : public abstract class FieldType extends FieldProperties { : public static final Logger log = Logger.getLogger(FieldType.class.getName()); : : - protected String typeName; // the name of the type, not the name of the field : - protected Map<String,String> args; // additional arguments : - protected int trueProperties; // properties explicitly set to true : - protected int falseProperties; // properties explicitly set to false : + /** The name of the type (not the name of the field) */ : + protected String typeName; : + /** additional arguments specified in the field type declaration */ : + protected Map<String,String> args; : + /** properties explicitly set to true */ : + protected int trueProperties; : + /** properties explicitly set to false */ : + protected int falseProperties; : int properties; : : + /** Returns true if fields of this type should be tokenized */ : protected boolean isTokenized() { : return (properties & TOKENIZED) != 0; : } : @@ -105,6 +110,7 @@ : } : } : : + /** :TODO: document this method */ : protected void restrictProps(int props) { : if ((properties & props) != 0) { : throw new RuntimeException("schema fieldtype " + typeName : @@ -113,7 +119,7 @@ : } : } : : - : + /** The Name of this FieldType as specified in the schema file */ : public String getTypeName() { : return typeName; : } : @@ -131,19 +137,31 @@ : } : : : - // used for adding a document when a field needs to be created from a type and a string : - // by default, the indexed value is the same as the stored value (taken from toInternal()) : - // Having a different representation for external, internal, and indexed would present quite : - // a few problems given the current Lucene architecture. An analyzer for adding docs would : - // need to translate internal->indexed while an analyzer for querying would need to : - // translate external->indexed. : - // : - // The only other alternative to having internal==indexed would be to have : - // internal==external. : - // In this case, toInternal should convert to the indexed representation, : - // toExternal() should do nothing, and createField() should *not* call toInternal, : - // but use the external value and set tokenized=true to get Lucene to convert : - // to the internal(indexed) form. : + /** : + * Used for adding a document when a field needs to be created from a : + * type and a string. : + * : + * <p> : + * By default, the indexed value is the same as the stored value : + * (taken from toInternal()). Having a different representation for : + * external, internal, and indexed would present quite a few problems : + * given the current Lucene architecture. An analyzer for adding docs : + * would need to translate internal->indexed while an analyzer for : + * querying would need to translate external->indexed. : + * </p> : + * <p> : + * The only other alternative to having internal==indexed would be to have : + * internal==external. In this case, toInternal should convert to : + * the indexed representation, toExternal() should do nothing, and : + * createField() should *not* call toInternal, but use the external : + * value and set tokenized=true to get Lucene to convert to the : + * internal(indexed) form. : + * </p> : + * : + * :TODO: clean up and clarify this explanation. : + * : + * @see #toInternal : + */ : public Field createField(SchemaField field, String externalVal, float boost) { : String val; : try { : @@ -162,31 +180,40 @@ : } : : : - // Convert an external value (from XML update command or from query string) : - // into the internal format. : - // - used in delete when a Term needs to be created. : - // - used by the default getTokenizer() and createField() : + /** : + * Convert an external value (from XML update command or from query string) : + * into the internal format. : + * @see #toExternal : + */ : public String toInternal(String val) { : + // - used in delete when a Term needs to be created. : + // - used by the default getTokenizer() and createField() : return val; : } : : - // Convert the stored-field format to an external (string, human readable) value : - // currently used in writing XML of the search result (but perhaps : - // a more efficient toXML(Field f, Writer w) should be used : - // in the future. : + /** : + * Convert the stored-field format to an external (string, human readable) : + * value : + * @see #toInternal : + */ : public String toExternal(Field f) { : + // currently used in writing XML of the search result (but perhaps : + // a more efficient toXML(Field f, Writer w) should be used : + // in the future. : return f.stringValue(); : } : : - : + /** :TODO: document this method */ : public String indexedToReadable(String indexedForm) { : return indexedForm; : } : : + /** :TODO: document this method */ : public String storedToReadable(Field f) { : return toExternal(f); : } : : + /** :TODO: document this method */ : public String storedToIndexed(Field f) { : // right now, the transformation of single valued fields like SortableInt : // is done when the Field is created, not at analysis time... this means : @@ -219,10 +246,10 @@ : **********/ : : : - // : - // Default analyzer for types that only produce 1 verbatim token... : - // A maximum size of chars to be read must be specified : - // : + /** : + * Default analyzer for types that only produce 1 verbatim token... : + * A maximum size of chars to be read must be specified : + */ : protected final class DefaultAnalyzer extends SolrAnalyzer { : final int maxChars; : : @@ -244,44 +271,77 @@ : } : : : - //analyzer set by schema for text types. : - //subclasses can set analyzer themselves or override getAnalyzer() : + /** : + * Analyzer set by schema for text types to use when indexing fields : + * of this type, subclasses can set analyzer themselves or override : + * getAnalyzer() : + * @see #getAnalyzer : + */ : protected Analyzer analyzer=new DefaultAnalyzer(256); : + : + /** : + * Analyzer set by schema for text types to use when searching fields : + * of this type, subclasses can set analyzer themselves or override : + * getAnalyzer() : + * @see #getQueryAnalyzer : + */ : protected Analyzer queryAnalyzer=analyzer; : : - // get analyzer should be fast to call... since the addition of dynamic fields, : - // this can be called all the time instead of just once at startup. : - // The analyzer will only be used in the following scenarios: : - // - during a document add for any field that has "tokenized" set (typically : - // only Text fields) : - // - during query parsing : - : + /** : + * Returns the Analyzer to be used when indexing fields of this type. : + * <p> : + * This method may be called many times, at any time. : + * </p> : + * @see #getQueryAnalyzer : + */ : public Analyzer getAnalyzer() { : return analyzer; : } : : + /** : + * Returns the Analyzer to be used when searching fields of this type. : + * <p> : + * This method may be called many times, at any time. : + * </p> : + * @see #getAnalyzer : + */ : public Analyzer getQueryAnalyzer() { : return queryAnalyzer; : } : : - // This is called by the schema parser if a custom analyzer is defined : + /** : + * Sets the Analyzer to be used when indexing fields of this type. : + * @see #getAnalyzer : + */ : public void setAnalyzer(Analyzer analyzer) { : this.analyzer = analyzer; : log.finest("FieldType: " + typeName + ".setAnalyzer(" + analyzer.getClass().getName() + ")" ); : } : : - // This is called by the schema parser if a custom analyzer is defined : + /** : + * Sets the Analyzer to be used when querying fields of this type. : + * @see #getQueryAnalyzer : + */ : public void setQueryAnalyzer(Analyzer analyzer) { : this.queryAnalyzer = analyzer; : log.finest("FieldType: " + typeName + ".setQueryAnalyzer(" + analyzer.getClass().getName() + ")" ); : } : : - : + /** : + * Renders the specified field as XML : + */ : public abstract void write(XMLWriter xmlWriter, String name, Field f) throws IOException; : : - : + : + /** : + * Returns the SortField instance that should be used to sort fields : + * of this type. : + */ : public abstract SortField getSortField(SchemaField field, boolean top); : : + /** : + * Utility usable by subclasses when they want to get basic String sorting. : + */ : protected SortField getStringSort(SchemaField field, boolean reverse) { : return Sorting.getStringSortField(field.name, reverse, field.sortMissingLast(),field.sortMissingFirst()); : } : : Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java : URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java?rev=412029&r1=412028&r2=412029&view=diff : ============================================================================== : --- incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java (original) : +++ incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java Tue Jun 6 00:53:40 2006 : @@ -56,11 +56,22 @@ : private String name; : private float version; : : + /** : + * Constructs a schema using the specified file name using the normal : + * Config path directory searching rules. : + * : + * @see Config#openResource : + */ : public IndexSchema(String schemaFile) { : this.schemaFile=schemaFile; : readConfig(); : } : : + /** : + * Direct acess to the InputStream for the schemaFile used by this instance. : + * : + * @see Config#openResource : + */ : public InputStream getInputStream() { : return Config.openResource(schemaFile); : } : @@ -70,39 +81,97 @@ : return version; : } : : + /** The Name of this schema (as specified in the schema file) */ : public String getName() { return name; } : : private final HashMap<String, SchemaField> fields = new HashMap<String,SchemaField>(); : private final HashMap<String, FieldType> fieldTypes = new HashMap<String,FieldType>(); : : + /** : + * Provides direct access to the Map containing all explicit : + * (ie: non-dynamic) fields in the index, keyed on field name. : + * : + * <p> : + * Modifying this Map (or any item in it) will affect the real schema : + * </p> : + */ : public Map<String,SchemaField> getFields() { return fields; } : + : + /** : + * Provides direct access to the Map containing all Field Types : + * in the index, keyed on fild type name. : + * : + * <p> : + * Modifying this Map (or any item in it) will affect the real schema : + * </p> : + */ : public Map<String,FieldType> getFieldTypes() { return fieldTypes; } : : : private Similarity similarity; : + : + /** : + * Returns the Similarity used for this index : + */ : public Similarity getSimilarity() { return similarity; } : : private Analyzer analyzer; : + : + /** : + * Returns the Analyzer used when indexing documents for this index : + * : + * <p> : + * This Analyzer is field (and dynamic field) name aware, and delegates to : + * a field specific Analyzer based on the field type. : + * </p> : + */ : public Analyzer getAnalyzer() { return analyzer; } : : private Analyzer queryAnalyzer; : + : + /** : + * Returns the Analyzer used when searching this index : + * : + * <p> : + * This Analyzer is field (and dynamic field) name aware, and delegates to : + * a field specific Analyzer based on the field type. : + * </p> : + */ : public Analyzer getQueryAnalyzer() { return queryAnalyzer; } : : private String defaultSearchFieldName=null; : + : + /** Name of the default search field specified in the schema file */ : public String getDefaultSearchFieldName() { : return defaultSearchFieldName; : } : : private SchemaField uniqueKeyField; : + : + /** : + * Unique Key field specified in the schema file : + * @return null if this schema has no unique key field : + */ : public SchemaField getUniqueKeyField() { return uniqueKeyField; } : : private String uniqueKeyFieldName; : private FieldType uniqueKeyFieldType; : : + /** : + * The raw (field type encoded) value of the Unique Key field for : + * the specified Document : + * @return null if this schema has no unique key field : + * @see #printableUniqueKey : + */ : public Field getUniqueKeyField(org.apache.lucene.document.Document doc) { : return doc.getField(uniqueKeyFieldName); // this should return null if name is null : } : : + /** : + * The printable value of the Unique Key field for : + * the specified Document : + * @return null if this schema has no unique key field : + */ : public String printableUniqueKey(org.apache.lucene.document.Document doc) { : Field f = doc.getField(uniqueKeyFieldName); : return f==null ? null : uniqueKeyFieldType.toExternal(f); : @@ -480,8 +549,14 @@ : : private DynamicField[] dynamicFields; : : - : - // get a field, and if not statically defined, check dynamic fields. : + /** : + * Returns the SchemaField that should be used for the specified field name : + * : + * @param fieldName may be an explicitly created field, or a name that : + * excercies a dynamic field. : + * @throws SolrException if no such field exists : + * @see #getFieldType : + */ : public SchemaField getField(String fieldName) { : SchemaField f = fields.get(fieldName); : if (f != null) return f; : @@ -498,8 +573,20 @@ : throw new SolrException(1,"undefined field "+fieldName); : } : : - // This method exists because it can be more efficient for dynamic fields : - // if a full SchemaField isn't needed. : + /** : + * Returns the FieldType for the specified field name. : + * : + * <p> : + * This method exists because it can be more efficient then : + * [EMAIL PROTECTED] #getField} for dynamic fields if a full SchemaField isn't needed. : + * </p> : + * : + * @param fieldName may be an explicitly created field, or a name that : + * excercies a dynamic field. : + * @throws SolrException if no such field exists : + * @see #getField(String) : + * @see #getFieldTypeNoEx : + */ : public FieldType getFieldType(String fieldName) { : SchemaField f = fields.get(fieldName); : if (f != null) return f.getType(); : @@ -508,8 +595,18 @@ : } : : /** : - * return null instead of throwing an exception if : - * the field is undefined. : + * Returns the FieldType for the specified field name. : + * : + * <p> : + * This method exists because it can be more efficient then : + * [EMAIL PROTECTED] #getField} for dynamic fields if a full SchemaField isn't needed. : + * </p> : + * : + * @param fieldName may be an explicitly created field, or a name that : + * excercies a dynamic field. : + * @return null if field is not defined. : + * @see #getField(String) : + * @see #getFieldTypeNoEx : */ : public FieldType getFieldTypeNoEx(String fieldName) { : SchemaField f = fields.get(fieldName); : @@ -518,6 +615,16 @@ : } : : : + /** : + * Returns the FieldType of the best matching dynamic field for : + * the specified field name : + * : + * @param fieldName may be an explicitly created field, or a name that : + * excercies a dynamic field. : + * @throws SolrException if no such field exists : + * @see #getField(String) : + * @see #getFieldTypeNoEx : + */ : public FieldType getDynamicFieldType(String fieldName) { : for (DynamicField df : dynamicFields) { : if (df.matches(fieldName)) return df.prototype.getType(); : @@ -534,6 +641,11 @@ : : : private final Map<String, SchemaField[]> copyFields = new HashMap<String,SchemaField[]>(); : + : + /** : + * Returns the list of fields that should recieve a copy of any indexed values added to the specified field. : + * @return may be null or empty if there are no matching copyField directives : + */ : public SchemaField[] getCopyFields(String sourceField) { : return copyFields.get(sourceField); : } : : Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java : URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java?rev=412029&r1=412028&r2=412029&view=diff : ============================================================================== : --- incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java (original) : +++ incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java Tue Jun 6 00:53:40 2006 : @@ -24,12 +24,11 @@ : import java.io.IOException; : : /** : + * Encapsulates all information about a Field in a Solr Schema : + * : * @author yonik : * @version $Id$ : */ : - : - : - : public final class SchemaField extends FieldProperties { : final String name; : final FieldType type; : : -Hoss
