Author: sshafroi Date: 2008-04-30 09:44:50 +0200 (Wed, 30 Apr 2008) New Revision: 6562
Modified: trunk/ trunk/generic.sesam/war/pom.xml trunk/query-api/src/main/java/no/sesat/search/query/token/RegExpEvaluatorFactory.java trunk/query-api/src/main/java/no/sesat/search/query/token/TokenPredicate.java trunk/query-api/src/main/java/no/sesat/search/query/token/VeryFastTokenEvaluator.java trunk/search-command-control-spi/src/main/java/no/sesat/search/mode/command/AbstractSearchCommand.java trunk/war/src/main/java/no/sesat/search/http/filters/SiteJspLoaderFilter.java trunk/war/src/main/java/no/sesat/search/http/servlet/BoomerangServlet.java Log: Merged revisions 6542-6561 via svnmerge from http://sesat.no/svn/sesat-kernel/branches/2.17 ........ r6542 | ssmiweve | 2008-04-23 14:52:13 +0200 (Wed, 23 Apr 2008) | 1 line SEARCH-4665 - NPE at TokenEvaluationEngineImpl.evaluateImpl ........ r6545 | ssmiweve | 2008-04-24 22:41:53 +0200 (Thu, 24 Apr 2008) | 1 line SEARCH-4668 - BoomerangServlet: String index out of range: -31 ........ r6546 | sshafroi | 2008-04-25 12:54:18 +0200 (Fri, 25 Apr 2008) | 8 lines SEARCH-4681 - When building sesat, the wrong "generic.sesam.war" is copied to $CATALINA_BASE. It is not fast-spesific even if I trig that profile. This is a known bug. The solution has been to build genericno afterwards since this build would update the "generic.sesam.war" with the correct fast-spesific file. This does not happen anymore. If I remove "generic.sesam.war" from $CATALINA_BASE, then the correct file is copied. Is there any problems with the timestamp check here? (Execute deploy goal in include-fast profile) ........ r6547 | ssmiweve | 2008-04-25 16:18:39 +0200 (Fri, 25 Apr 2008) | 2 lines SEARCH-4663 - SiteJspLoaderFilter: java.nio.channels.OverlappingFileLockException appears i misunderstood the java.nio.channels package a little, at least the locking part of it. ........ r6548 | ssmiweve | 2008-04-25 16:22:07 +0200 (Fri, 25 Apr 2008) | 2 lines SEARCH-4665 - NPE at TokenEvaluationEngineImpl.evaluateImpl ........ r6560 | ssmiweve | 2008-04-29 19:32:17 +0200 (Tue, 29 Apr 2008) | 1 line SEARCH-3982 - Sortby: word [space] [hyphen] [space] word ........ r6561 | sshafroi | 2008-04-30 09:28:51 +0200 (Wed, 30 Apr 2008) | 1 line Make sure we add the include-fast classifier when in the include-fast profile ........ Property changes on: trunk ___________________________________________________________________ Name: svnmerge-integrated - /branches/2.10:1-4690,4692-4745 /branches/2.11:1-4933 /branches/2.12:1-5051,5053-5106 /branches/2.13:1-5378 /branches/2.14:1-5508 /branches/2.15:1-5995 /branches/2.16:1-6499 /branches/2.17:1-6541 /branches/2.6:1-3877 /branches/2.7:1-4160 /branches/2.8:1-4446 /branches/2.9:1-4626 /branches/MAP_SEARCHv2:1-4544 + /branches/2.10:1-4690,4692-4745 /branches/2.11:1-4933 /branches/2.12:1-5051,5053-5106 /branches/2.13:1-5378 /branches/2.14:1-5508 /branches/2.15:1-5995 /branches/2.16:1-6499 /branches/2.17:1-6561 /branches/2.6:1-3877 /branches/2.7:1-4160 /branches/2.8:1-4446 /branches/2.9:1-4626 /branches/MAP_SEARCHv2:1-4544 Modified: trunk/generic.sesam/war/pom.xml =================================================================== --- trunk/generic.sesam/war/pom.xml 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/generic.sesam/war/pom.xml 2008-04-30 07:44:50 UTC (rev 6562) @@ -90,7 +90,29 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-war-plugin</artifactId> <configuration><classifier>include-fast</classifier></configuration> - </plugin> + </plugin> + <plugin> + <groupId>sesat</groupId> + <artifactId>sesat-mojo</artifactId> + + <configuration> + <artifactItems> + <artifactItem> + <classifier>include-fast</classifier> + </artifactItem> + </artifactItems> + </configuration> + + <executions> + <execution> + <id>copy-self</id> + <phase>install</phase> + <goals> + <goal>deploy</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> <dependencies> Modified: trunk/query-api/src/main/java/no/sesat/search/query/token/RegExpEvaluatorFactory.java =================================================================== --- trunk/query-api/src/main/java/no/sesat/search/query/token/RegExpEvaluatorFactory.java 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/query-api/src/main/java/no/sesat/search/query/token/RegExpEvaluatorFactory.java 2008-04-30 07:44:50 UTC (rev 6562) @@ -135,9 +135,16 @@ final String tokenName = evaluator.getAttribute("token"); LOG.info(" ->[EMAIL PROTECTED]: " + tokenName); - final TokenPredicate token = null != TokenPredicate.Static.getTokenPredicate(tokenName) - ? TokenPredicate.Static.getTokenPredicate(tokenName) - : TokenPredicate.Static.createAnonymousTokenPredicate(tokenName, TokenPredicate.Type.REGEX); + TokenPredicate token; + try{ + token = TokenPredicate.Static.getTokenPredicate(tokenName); + + }catch(IllegalArgumentException iae){ + LOG.debug(tokenName + " does not exist. Will create it. Underlying exception was " + iae); + token = TokenPredicate.Static.createAnonymousTokenPredicate( + tokenName, + TokenPredicate.Type.REGEX); + } final boolean queryDep = Boolean.parseBoolean(evaluator.getAttribute("query-dependant")); LOG.info(" ->[EMAIL PROTECTED]: " + queryDep); Modified: trunk/query-api/src/main/java/no/sesat/search/query/token/TokenPredicate.java =================================================================== --- trunk/query-api/src/main/java/no/sesat/search/query/token/TokenPredicate.java 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/query-api/src/main/java/no/sesat/search/query/token/TokenPredicate.java 2008-04-30 07:44:50 UTC (rev 6562) @@ -44,7 +44,7 @@ * @todo will need to become a class that can be extended. SEARCH-3540. a mapping to the Evaluation implementation. */ public static final class Type implements Serializable{ - + public static final Type FAST = new Type("FAST", VeryFastTokenEvaluator.class); public static final Type REGEX = new Type("REGEX", RegExpTokenEvaluator.class); public static final Type JEP = new Type("JEP", JepTokenEvaluator.class); @@ -61,7 +61,7 @@ } /** The name of the TokenPredicate. Must be uppercase. Must be unique across all skins. - * + * * @return TokenPredicate name. */ String name(); @@ -71,7 +71,7 @@ * @return the type */ Type getType(); - + /** * Evaluates to true if fastListName occurs in the query. This method uses a * TokenEvaluationEngine to get a TokenEvaluator. @@ -85,9 +85,9 @@ * TokTokenEvaluationEngineastListName evaluates to true. */ boolean evaluate(Object evalFactory); - - - + + + // Inner Classes ----------------------------------------------------- /** A formalised breakdown of metadata categories that search terms can match. @@ -447,9 +447,9 @@ return TokenPredicateImpl.evaluate(this, evalFactory); } - - + + } /** The default implementation used. Should not be used directly. @@ -459,30 +459,30 @@ static class TokenPredicateImpl implements TokenPredicate{ // Constants ----------------------------------------------------- - + private static final String ERR_ARG_NOT_TOKEN_EVALUATOR_FACTORY = "Argument to evaluate must be an instance of a TokenEvaluationEngine"; private static final String ERR_METHOD_CLOSED_TO_OTHER_THREADS = "TokenPredicate.evaluate(..) can only be used by same thread that created TokenEvaluationEngine!"; private static final String ERR_ENGINE_MISSING_STATE = "TokenEvaluationEngine must have state assigned"; - - private static final Set<TokenPredicate> TOKENS + + private static final Set<TokenPredicate> TOKENS = new CopyOnWriteArraySet<TokenPredicate>(); - private static final Map<Type,Set<TokenPredicate>> TOKENS_BY_TYPE + private static final Map<Type,Set<TokenPredicate>> TOKENS_BY_TYPE = new ConcurrentHashMap<Type,Set<TokenPredicate>>(); - + /** @deprecated todo take out of sesat. **/ private static final Set<TokenPredicate> MAGIC_TOKENS = new CopyOnWriteArraySet<TokenPredicate>(); /** @deprecated todo take out of sesat. **/ private static final Set<TokenPredicate> TRIGGER_TOKENS = new CopyOnWriteArraySet<TokenPredicate>(); - + // Attributes ----------------------------------------------------- - + private final String name; private final Type type; // Constructors ----------------------------------------------------- - + private TokenPredicateImpl(final String name, final Type type){ this.name = name; this.type = type; @@ -502,8 +502,16 @@ TOKENS_BY_TYPE.get(type).add(this); } + + // public ----------------------------------------------------- + + @Override + public String toString() { + return name(); + } + // TokenPredicate implementation ------------------------------------ - + public String name(){ return name; } @@ -511,14 +519,14 @@ public Type getType(){ return type; } - + public boolean evaluate(final Object evalFactory) { return TokenPredicateImpl.evaluate(this, evalFactory); } - - // private ----------------------------------------------------- - + + // private ----------------------------------------------------- + private static boolean evaluate(final TokenPredicate token, final Object evalFactory) { // pre-condition checks @@ -578,14 +586,14 @@ super(vflqe); } } - + /** Utility class providing all useful static methods around TokenPredicates. * @todo move out to TokenPredicateUtility. **/ static final class Static{ private static final Map<String,TokenPredicate> ANONYMOUS_TOKENS = new ConcurrentHashMap<String,TokenPredicate>(); - + static{ // ensures all the enums have been loaded before any of the following static methods are called. // offspin to this is that there can be no references back to Static from Categories or TokenPredicateImpl. @@ -596,8 +604,9 @@ * * @param name the name of the TokenPredicate to find. * @return the TokenPredicate. + * @throws IllegalArgumentException when no such anonymous token by the name exists. */ - public static TokenPredicate getTokenPredicate(final String name){ + public static TokenPredicate getTokenPredicate(final String name) throws IllegalArgumentException{ try{ return Categories.valueOf(name); @@ -610,9 +619,13 @@ * * @param name the name of the TokenPredicate to find. * @return the anonymous TokenPredicate. + * @throws IllegalArgumentException when no such anonymous token by the name exists. */ - public static TokenPredicate getAnonymousTokenPredicate(final String name){ + public static TokenPredicate getAnonymousTokenPredicate(final String name) throws IllegalArgumentException{ + if(!ANONYMOUS_TOKENS.containsKey(name)){ + throw new IllegalArgumentException("No anonymous token found with name " + name); + } return ANONYMOUS_TOKENS.get(name); } @@ -661,6 +674,6 @@ public static Set<TokenPredicate> getTriggerTokenPredicates() { return Collections.unmodifiableSet(TokenPredicateImpl.TRIGGER_TOKENS); } - } + } } \ No newline at end of file Modified: trunk/query-api/src/main/java/no/sesat/search/query/token/VeryFastTokenEvaluator.java =================================================================== --- trunk/query-api/src/main/java/no/sesat/search/query/token/VeryFastTokenEvaluator.java 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/query-api/src/main/java/no/sesat/search/query/token/VeryFastTokenEvaluator.java 2008-04-30 07:44:50 UTC (rev 6562) @@ -320,9 +320,16 @@ final String tokenName = list.getAttribute("token"); LOG.info(" ->[EMAIL PROTECTED]: " + tokenName); - final TokenPredicate token = null != TokenPredicate.Static.getTokenPredicate(tokenName) - ? TokenPredicate.Static.getTokenPredicate(tokenName) - : TokenPredicate.Static.createAnonymousTokenPredicate(tokenName, TokenPredicate.Type.FAST); + TokenPredicate token; + try{ + token = TokenPredicate.Static.getTokenPredicate(tokenName); + + }catch(IllegalArgumentException iae){ + LOG.debug(tokenName + " does not exist. Will create it. Underlying exception was " + iae); + token = TokenPredicate.Static.createAnonymousTokenPredicate( + tokenName, + TokenPredicate.Type.FAST); + } final String[] listNameArr = list.getAttribute("list-name").split(","); LOG.info(" ->lists: " + list.getAttribute("list-name")); @@ -526,10 +533,11 @@ // Also remove any operator characters. (SEARCH-3883 & SEARCH-3967) return string - .replaceAll(" ", "xxKEEPWSxx") // Hack to keep spaces. + .replaceAll(" ", "xxKEEPWSxx") // Hack to keep spaces. multiple spaces always normalised. .replaceAll(SKIP_REGEX, " ") .replaceAll("xxKEEPWSxx", " ") // Hack to keep spaces. - .replaceAll(OPERATOR_REGEX, " "); + .replaceAll(OPERATOR_REGEX, " ") + .replaceAll(" +", " "); // normalise } // Inner classes ------------------------------------------------- Modified: trunk/search-command-control-spi/src/main/java/no/sesat/search/mode/command/AbstractSearchCommand.java =================================================================== --- trunk/search-command-control-spi/src/main/java/no/sesat/search/mode/command/AbstractSearchCommand.java 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/search-command-control-spi/src/main/java/no/sesat/search/mode/command/AbstractSearchCommand.java 2008-04-30 07:44:50 UTC (rev 6562) @@ -818,8 +818,10 @@ // if the field is the token then mask the field and include the term. // XXX why are we checking the known and possible predicates? boolean result = clause.getKnownPredicates().contains(tp); - result |= clause.getPossiblePredicates().contains(tp); - result &= engine.evaluateTerm(tp, clause.getField()); + + result |= clause.getPossiblePredicates().contains(tp) + && engine.evaluateTerm(tp, clause.getField()); + if (result) { field = fieldFilter; break; Modified: trunk/war/src/main/java/no/sesat/search/http/filters/SiteJspLoaderFilter.java =================================================================== --- trunk/war/src/main/java/no/sesat/search/http/filters/SiteJspLoaderFilter.java 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/war/src/main/java/no/sesat/search/http/filters/SiteJspLoaderFilter.java 2008-04-30 07:44:50 UTC (rev 6562) @@ -1,4 +1,4 @@ -/* Copyright (2006-2007) Schibsted Søk AS +/* Copyright (2006-2008) Schibsted Søk AS * This file is part of SESAT. * * SESAT is free software: you can redistribute it and/or modify @@ -85,14 +85,12 @@ private FilterConfig config; private String root; - /** [EMAIL PROTECTED] **/ public void init(final FilterConfig filterConfig) throws ServletException { config = filterConfig; root = config.getServletContext().getRealPath("/"); } - /** [EMAIL PROTECTED] **/ public void doFilter( final ServletRequest request, final ServletResponse response, @@ -110,7 +108,6 @@ chain.doFilter(request, response); } - /** [EMAIL PROTECTED] **/ public void destroy() { } @@ -204,23 +201,32 @@ final FileChannel channel = fileAccess.getChannel(); try{ - channel.lock(); + // channel.lock() only synchronises file access between programs, but not between threads inside + // the current JVM. The latter results in the OverlappingFileLockException. + // At least this is my current understanding of java.nio.channels + // It may be that no synchronisation or locking is required at all. A beer to whom answers :-) + // So we must provide synchronisation between our own threads, + // synchronisation against the file's path (using the JVM's String.intern() functionality) + // should work. (I can't imagine this string be used for any other synchronisation purposes). + synchronized(file.toString().intern()){ + channel.lock(); - if(fileExisted){ + if(fileExisted){ - final byte[] bytes = new byte[(int)channel.size()]; - final ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); - int reads; do{ reads = channel.read(byteBuffer); }while(0 < reads); + final byte[] bytes = new byte[(int)channel.size()]; + final ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + int reads; do{ reads = channel.read(byteBuffer); }while(0 < reads); - needsUpdating = !Arrays.equals(golden, bytes); - } + needsUpdating = !Arrays.equals(golden, bytes); + } - if(needsUpdating){ - // download file from skin - channel.write(ByteBuffer.wrap(golden), 0); - channel.force(true); - file.deleteOnExit(); + if(needsUpdating){ + // download file from skin + channel.write(ByteBuffer.wrap(golden), 0); + channel.force(true); + file.deleteOnExit(); + } } }finally{ channel.close(); Modified: trunk/war/src/main/java/no/sesat/search/http/servlet/BoomerangServlet.java =================================================================== --- trunk/war/src/main/java/no/sesat/search/http/servlet/BoomerangServlet.java 2008-04-30 07:28:51 UTC (rev 6561) +++ trunk/war/src/main/java/no/sesat/search/http/servlet/BoomerangServlet.java 2008-04-30 07:44:50 UTC (rev 6562) @@ -1,4 +1,4 @@ -/* +/* * Copyright (2006-2007) Schibsted Søk AS * This file is part of SESAT. * @@ -37,7 +37,11 @@ import org.apache.log4j.Logger; /** - * Logs different statistics with ajax + * Provides the user-statistics logging in Sesat. + * Links are logged with <b>ceremonial</b> boomerangs that come back (ie with a redirect response). + * Javascript functionality (or user behavour) is logged with <b>hunting</b> boomerangs that do not come back. + * + * A cermonial example is: * * @author <a href="mailto:[EMAIL PROTECTED]">Thomas Kjaerstad</a> * @author <a href="mailto:[EMAIL PROTECTED]">Mck</a> @@ -75,37 +79,44 @@ } // pick out the entrails - final String grub = url.substring( - url.indexOf(CEREMONIAL) + CEREMONIAL.length() + 1, - url.indexOf("/", url.indexOf(CEREMONIAL) + CEREMONIAL.length() + 1)); - LOG.debug(grub); + final int boomerangStart = url.indexOf(CEREMONIAL) + CEREMONIAL.length() + 1; - // the url to return to - final String destination = url.substring( - url.indexOf("/", url.indexOf(CEREMONIAL) + CEREMONIAL.length() + 1) + 1); + try{ + final String grub = url.substring(boomerangStart, url.indexOf("/", boomerangStart)); + LOG.debug(grub); - // grub it up - final Map<String,String> entrails = new HashMap<String,String>(); - if(0 < grub.length()){ - final StringTokenizer tokeniser = new StringTokenizer(grub, ";"); - while(tokeniser.hasMoreTokens()){ - final String[] entry = tokeniser.nextToken().split("="); - entrails.put(entry[0], 1 < entry.length ? entry[1] : entry[0]); + // the url to return to + final String destination = url.substring( + url.indexOf("/", url.indexOf(CEREMONIAL) + CEREMONIAL.length() + 1) + 1); + + // grub it up + final Map<String,String> entrails = new HashMap<String,String>(); + if(0 < grub.length()){ + final StringTokenizer tokeniser = new StringTokenizer(grub, ";"); + while(tokeniser.hasMoreTokens()){ + final String[] entry = tokeniser.nextToken().split("="); + entrails.put(entry[0], 1 < entry.length ? entry[1] : entry[0]); + } } - } - entrails.put("boomerang", destination); - kangerooGrub(entrails); + entrails.put("boomerang", destination); + kangerooGrub(entrails); - LOG.debug("Ceremonial boomerang to " + destination.toString()); - if(req.getHeader("User-agent").matches("(Googlebot|Slurp|Yahoo\\! Slurp)")){ - // crawlers like permanent redirects. and we're not interested in their clicks so ok to cache. - res.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); - res.setHeader("Location", destination.toString()); - res.setHeader("Connection", "close"); - - }else{ - // default behaviour for users. - res.sendRedirect(destination.toString()); + LOG.debug("Ceremonial boomerang to " + destination.toString()); + if(req.getHeader("User-agent").matches("(Googlebot|Slurp|Yahoo\\! Slurp)")){ + // crawlers like permanent redirects. and we're not interested in their clicks so ok to cache. + res.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + res.setHeader("Location", destination.toString()); + res.setHeader("Connection", "close"); + + }else{ + // default behaviour for users. + res.sendRedirect(destination.toString()); + } + + }catch(StringIndexOutOfBoundsException sioobe){ + // SEARCH-4668 + LOG.error("Boomerang url not to standard --> " + url); + LOG.debug(sioobe.getMessage(), sioobe); } }else{ @@ -129,8 +140,8 @@ for(String key : paramKeys){ try { - final String value = params.get(key) instanceof StringDataObject - ? ((StringDataObject) params.get(key)).getXmlEscaped() + final String value = params.get(key) instanceof StringDataObject + ? ((StringDataObject) params.get(key)).getXmlEscaped() : StringEscapeUtils.escapeXml((String) params.get(key)); // it's critical for the logparser that we write valid xml _______________________________________________ Kernel-commits mailing list [email protected] http://sesat.no/mailman/listinfo/kernel-commits
