Author: ssmiweve
Date: 2009-09-30 16:43:54 +0200 (Wed, 30 Sep 2009)
New Revision: 7299
Added:
trunk/generic.sesam/search-command-config/src/main/java/no/sesat/search/mode/config/GoogleCommandConfig.java
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestful.java
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestfulSearchCommand.java
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/GoogleSearchCommand.java
Modified:
trunk/generic.sesam/search-command-control/default/pom.xml
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlRestful.java
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlSearchCommand.java
trunk/generic.sesam/sesam.com/war/src/main/conf/modes.xml
trunk/generic.sesam/war/src/main/conf/configuration.properties
trunk/generic.sesam/war/src/main/conf/modes.xml
Log:
Google search command
Added:
trunk/generic.sesam/search-command-config/src/main/java/no/sesat/search/mode/config/GoogleCommandConfig.java
===================================================================
---
trunk/generic.sesam/search-command-config/src/main/java/no/sesat/search/mode/config/GoogleCommandConfig.java
(rev 0)
+++
trunk/generic.sesam/search-command-config/src/main/java/no/sesat/search/mode/config/GoogleCommandConfig.java
2009-09-30 14:43:54 UTC (rev 7299)
@@ -0,0 +1,42 @@
+/* Copyright (2009) Schibsted ASA
+ * This file is part of SESAT.
+ *
+ * SESAT is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SESAT is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with SESAT. If not, see <http://www.gnu.org/licenses/>.
+ */
+package no.sesat.search.mode.config;
+
+import no.sesat.search.mode.config.CommandConfig.Controller;
+
+/**
+ * Search against google search API
+ *
http://code.google.com/apis/youtube/developers_guide_protocol.html#Searching_for_Videos
+ */
+...@controller("GoogleSearchCommand")
+public class GoogleCommandConfig extends AbstractRestfulSearchConfiguration {
+
+ private String path = "googleWebPath";
+
+ public boolean getLargeResults() {
+ return 4 < getResultsToReturn();
+ }
+
+ public String getPath(){
+ return path;
+ }
+
+ public void setPath(final String path){
+ this.path = path;
+ }
+
+}
Property changes on:
trunk/generic.sesam/search-command-config/src/main/java/no/sesat/search/mode/config/GoogleCommandConfig.java
___________________________________________________________________
Added: svn:keywords
+ Id
Modified: trunk/generic.sesam/search-command-control/default/pom.xml
===================================================================
--- trunk/generic.sesam/search-command-control/default/pom.xml 2009-09-30
12:34:54 UTC (rev 7298)
+++ trunk/generic.sesam/search-command-control/default/pom.xml 2009-09-30
14:43:54 UTC (rev 7299)
@@ -116,6 +116,11 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.sf.json-lib</groupId>
+ <artifactId>json-lib</artifactId>
+ <classifier>jdk15</classifier>
+ </dependency>
<!-- testin -->
<dependency>
Added:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestful.java
===================================================================
---
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestful.java
(rev 0)
+++
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestful.java
2009-09-30 14:43:54 UTC (rev 7299)
@@ -0,0 +1,93 @@
+/*
+ * Copyright (2006-2007) Schibsted ASA
+ * This file is part of SESAT.
+ *
+ * SESAT is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SESAT is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with SESAT. If not, see <http://www.gnu.org/licenses/>.
+ */
+package no.sesat.search.mode.command;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import no.sesat.search.http.HTTPClient;
+import no.sesat.search.mode.command.SearchCommand.Context;
+import no.sesat.search.mode.config.AbstractRestfulSearchConfiguration;
+import no.sesat.search.site.config.SiteConfiguration;
+import org.apache.log4j.Logger;
+
+/**
+ *
+ * @version $Id$
+ */
+public abstract class AbstractRestful implements Restful{
+
+ // Constants -----------------------------------------------------
+
+ //private static final Logger LOG =
Logger.getLogger(AbstractXmlSearchCommand.class);
+ private static final Logger DUMP =
Logger.getLogger("no.sesat.search.Dump");
+
+ // Attributes ----------------------------------------------------
+
+ private final Context context;
+
+ private final transient HTTPClient client;
+
+ // Static --------------------------------------------------------
+
+ // Constructors --------------------------------------------------
+
+ /**
+ * Create new restful
+ *
+ * @param cxt The context to execute in.
+ */
+ public AbstractRestful(final Context cxt) {
+
+ context = cxt;
+
+ final AbstractRestfulSearchConfiguration conf =
(AbstractRestfulSearchConfiguration)cxt.getSearchConfiguration();
+
+ final SiteConfiguration siteConf =
cxt.getDataModel().getSite().getSiteConfiguration();
+ final String host = siteConf.getProperty(conf.getHost());
+ final int port = null != siteConf.getProperty(conf.getPort())
+ ? Integer.parseInt(siteConf.getProperty(conf.getPort()))
+ : 80; // defaults to normal http port
+
+ client = HTTPClient.instance(conf.getHostHeader().length() > 0 ?
conf.getHostHeader() : host, port, host);
+ }
+
+ // Public --------------------------------------------------------
+
+ public Context getContext() {
+ return context;
+ }
+
+ @Override
+ public final BufferedReader getHttpReader(final String encoding) throws
IOException {
+ final String url = createRequestURL();
+ AbstractRestful.DUMP.info("Using " + url);
+ return client.getBufferedReader(url, encoding);
+ }
+
+
+ // Protected -------------------------------------------------------
+
+ protected HTTPClient getClient(){
+ return client;
+ }
+
+ // Private -------------------------------------------------------
+
+ // Inner classes -------------------------------------------------
+
+}
Property changes on:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestful.java
___________________________________________________________________
Added: svn:keywords
+ Id
Added:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestfulSearchCommand.java
===================================================================
---
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestfulSearchCommand.java
(rev 0)
+++
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestfulSearchCommand.java
2009-09-30 14:43:54 UTC (rev 7299)
@@ -0,0 +1,77 @@
+/*
+ * Copyright (2006-2009) Schibsted ASA
+ * This file is part of SESAT.
+ *
+ * SESAT is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SESAT is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with SESAT. If not, see <http://www.gnu.org/licenses/>.
+ */
+package no.sesat.search.mode.command;
+
+/**
+ * Helper base implementation for search commands that are RESTful.
+ *
+ * The RESTful server is defined through:
+ * host: AbstractRestfulSearchConfiguration.getHost()
+ * port: AbstractRestfulSearchConfiguration.getPort()
+ *
+ * @version $Id$
+ */
+public abstract class AbstractRestfulSearchCommand extends
AbstractSearchCommand{
+
+
+ // Constants -----------------------------------------------------
+
+ //private static final Logger LOG =
Logger.getLogger(AbstractXmlSearchCommand.class);
+
+ // Attributes ----------------------------------------------------
+
+ private Restful restful;
+
+ // Static --------------------------------------------------------
+
+ // Constructors --------------------------------------------------
+
+
+ /**
+ * Create new xml based command.
+ *
+ * @param cxt The context to execute in.
+ */
+ protected AbstractRestfulSearchCommand(final Context cxt) {
+ super(cxt);
+ }
+
+ // Public --------------------------------------------------------
+
+ public String createRequestURL() {
+
+ return restful.createRequestURL();
+ }
+
+
+
+ // Protected -----------------------------------------------------
+
+ protected final Restful getRestful(){
+ return restful;
+ }
+
+ protected final void setRestful(final Restful restful){
+ this.restful = restful;
+ }
+
+ // Private -------------------------------------------------------
+
+ // Inner classes -------------------------------------------------
+
+}
Property changes on:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractRestfulSearchCommand.java
___________________________________________________________________
Added: svn:keywords
+ Id
Modified:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlRestful.java
===================================================================
---
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlRestful.java
2009-09-30 12:34:54 UTC (rev 7298)
+++
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlRestful.java
2009-09-30 14:43:54 UTC (rev 7299)
@@ -1,5 +1,5 @@
/*
- * Copyright (2006-2007) Schibsted ASA
+ * Copyright (2006-2009) Schibsted ASA
* This file is part of SESAT.
*
* SESAT is free software: you can redistribute it and/or modify
@@ -38,7 +38,7 @@
*
* @version $Id$
*/
-public abstract class AbstractXmlRestful implements XmlRestful{
+public abstract class AbstractXmlRestful extends AbstractRestful implements
XmlRestful{
// Constants -----------------------------------------------------
@@ -48,55 +48,31 @@
// Attributes ----------------------------------------------------
- private final Context context;
-
- private final transient HTTPClient client;
-
// Static --------------------------------------------------------
// Constructors --------------------------------------------------
/**
- * Create new xml based command.
+ * Create new XmlRestful
*
* @param cxt The context to execute in.
*/
public AbstractXmlRestful(final Context cxt) {
- context = cxt;
-
- final AbstractXmlSearchConfiguration conf =
(AbstractXmlSearchConfiguration)cxt.getSearchConfiguration();
-
- final SiteConfiguration siteConf =
cxt.getDataModel().getSite().getSiteConfiguration();
- final String host = siteConf.getProperty(conf.getHost());
- final int port = null != siteConf.getProperty(conf.getPort())
- ? Integer.parseInt(siteConf.getProperty(conf.getPort()))
- : 80; // defaults to normal http port
-
- client = HTTPClient.instance(conf.getHostHeader().length() > 0 ?
conf.getHostHeader() : host, port, host);
+ super(cxt);
}
// Public --------------------------------------------------------
+ @Override
public final Document getXmlResult() throws IOException, SAXException {
- final String url = createRequestURL();
- DUMP.info("Using " + url);
- return client.getXmlDocument(url);
- }
- public final BufferedReader getHttpReader(final String encoding) throws
IOException {
final String url = createRequestURL();
DUMP.info("Using " + url);
- return client.getBufferedReader(url, encoding);
+ return getClient().getXmlDocument(url);
}
- // Protected -----------------------------------------------------
-
- public Context getContext(){
- return context;
- }
-
// Private -------------------------------------------------------
// Inner classes -------------------------------------------------
Modified:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlSearchCommand.java
===================================================================
---
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlSearchCommand.java
2009-09-30 12:34:54 UTC (rev 7298)
+++
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/AbstractXmlSearchCommand.java
2009-09-30 14:43:54 UTC (rev 7299)
@@ -1,5 +1,5 @@
/*
- * Copyright (2006-2008) Schibsted ASA
+ * Copyright (2006-2009) Schibsted ASA
* This file is part of SESAT.
*
* SESAT is free software: you can redistribute it and/or modify
@@ -31,7 +31,7 @@
*
* @version $Id$
*/
-public abstract class AbstractXmlSearchCommand extends AbstractSearchCommand{
+public abstract class AbstractXmlSearchCommand extends
AbstractRestfulSearchCommand{
// Constants -----------------------------------------------------
@@ -58,13 +58,6 @@
// Public --------------------------------------------------------
- public String createRequestURL() {
-
- return restful.createRequestURL();
- }
-
-
-
// Protected -----------------------------------------------------
Added:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/GoogleSearchCommand.java
===================================================================
---
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/GoogleSearchCommand.java
(rev 0)
+++
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/GoogleSearchCommand.java
2009-09-30 14:43:54 UTC (rev 7299)
@@ -0,0 +1,145 @@
+/*
+ * Copyright (2009) Schibsted ASA
+ * This file is part of SESAT.
+ *
+ * SESAT is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SESAT is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with SESAT. If not, see <http://www.gnu.org/licenses/>.
+
+ */
+package no.sesat.search.mode.command;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.SocketTimeoutException;
+import java.text.MessageFormat;
+import java.util.List;
+import no.sesat.search.result.BasicResultItem;
+import no.sesat.search.result.BasicResultList;
+import no.sesat.search.result.ResultItem;
+import no.sesat.search.result.ResultList;
+import net.sf.json.JSONSerializer;
+import no.sesat.search.mode.config.GoogleCommandConfig;
+import org.apache.commons.beanutils.DynaBean;
+import org.apache.commons.beanutils.DynaProperty;
+import org.apache.log4j.Logger;
+
+/**
+ * @xxx very little has been done on this
+ * @todo extract out a AbstractJsonSearchCommand
+ * @see GoogleCommandConfig
+ * @version $Id$
+ */
+public class GoogleSearchCommand extends AbstractRestfulSearchCommand {
+
+ // Constants -----------------------------------------------------
+
+ private static final Logger LOG =
Logger.getLogger(GoogleSearchCommand.class);
+
+ private static final String COMMAND_URL_QUERY =
"?q={0}&rsz={1}&start={2}&v=1.0";
+
+ // Static --------------------------------------------------------
+ // Attributes ----------------------------------------------------
+
+ private final Context cxt;
+ private final GoogleCommandConfig conf;
+
+ // Constructors -------------------------------------------------
+
+ public GoogleSearchCommand(final Context cxt) {
+ super(cxt);
+ this.cxt = cxt;
+ conf = (GoogleCommandConfig) cxt.getSearchConfiguration();
+ final String commandPath =
cxt.getDataModel().getSite().getSiteConfiguration().getProperty(conf.getPath());
+ setRestful(
+ new AbstractRestful(cxt) {
+ @Override
+ public String createRequestURL() {
+
+ return MessageFormat.format(
+ commandPath + COMMAND_URL_QUERY,
+
cxt.getDataModel().getQuery().getUtf8UrlEncoded(),
+ conf.getLargeResults() ? "large" : "small",
+ GoogleSearchCommand.this.getOffset());
+ }
+ });
+ }
+
+ // Public --------------------------------------------------------
+
+ @Override
+ public ResultList<ResultItem> execute() {
+
+ final ResultList<ResultItem> result = new
BasicResultList<ResultItem>();
+ final BufferedReader reader;
+ try {
+
+ reader = getRestful().getHttpReader("UTF8");
+
+
+ if(null != reader) {
+
+ final StringBuilder builder = new StringBuilder();
+ String line;
+ while(null != (line = reader.readLine())){
+ builder.append(line);
+ }
+
+ final DynaBean bean = (DynaBean)
JSONSerializer.toJava(JSONSerializer.toJSON(builder.toString()));
+ final DynaBean data = (DynaBean)bean.get("responseData");
+
+ final String totalResults =
((DynaBean)data.get("cursor")).get("estimatedResultCount").toString();
+ result.setHitCount(Integer.parseInt(totalResults));
+
+ final List<DynaBean> results =
(List<DynaBean>)data.get("results");
+
+ for(DynaBean r : results) {
+
+ result.addResult(createItem(r));
+ }
+
+ }
+ } catch (SocketTimeoutException ste) {
+ LOG.error(getSearchConfiguration().getName() + " --> " +
ste.getMessage());
+ return new BasicResultList<ResultItem>();
+ } catch (IOException ex) {
+ throw new SearchCommandException(ex);
+ }
+ return result;
+ }
+
+ // Protected --------------------------------------------------------
+
+ protected ResultItem createItem(final DynaBean entry) {
+
+ ResultItem resItem = new BasicResultItem();
+
+ final DynaProperty[] descriptors =
entry.getDynaClass().getDynaProperties();
+ for(DynaProperty d : descriptors){
+ if(entry.get(d.getName()) instanceof Serializable){
+ resItem = resItem.addObjectField(d.getName(), (Serializable)
entry.get(d.getName()));
+ }
+ }
+
+ return resItem;
+
+ }
+
+ @Override
+ protected int getOffset() {
+ return super.getOffset();
+ }
+
+ // Private --------------------------------------------------------
+
+}
\ No newline at end of file
Property changes on:
trunk/generic.sesam/search-command-control/default/src/main/java/no/sesat/search/mode/command/GoogleSearchCommand.java
___________________________________________________________________
Added: svn:keywords
+ Id
Modified: trunk/generic.sesam/sesam.com/war/src/main/conf/modes.xml
===================================================================
--- trunk/generic.sesam/sesam.com/war/src/main/conf/modes.xml 2009-09-30
12:34:54 UTC (rev 7298)
+++ trunk/generic.sesam/sesam.com/war/src/main/conf/modes.xml 2009-09-30
14:43:54 UTC (rev 7299)
@@ -53,6 +53,8 @@
<!--yahoo-web-command id="globalSearch"
inherit="default-yahoo-web-command"/-->
<yahoo-boss-command id="globalSearch"
inherit="default-yahoo-boss-command"/>
+ <!--google-command id="globalSearch"
inherit="default-google-web-command"/-->
+
<!--solr-command id="globalSearch" inherit="default-command"
result-fields="list_name AS title,list_entry AS
body"
server-url="solr.serverUrl"/-->
Modified: trunk/generic.sesam/war/src/main/conf/configuration.properties
===================================================================
--- trunk/generic.sesam/war/src/main/conf/configuration.properties
2009-09-30 12:34:54 UTC (rev 7298)
+++ trunk/generic.sesam/war/src/main/conf/configuration.properties
2009-09-30 14:43:54 UTC (rev 7299)
@@ -47,6 +47,18 @@
yahooBossHost=boss.yahooapis.com
yahooBossPort=80
+# Google Search API
+googleHost=ajax.googleapis.com
+googlePort=80
+googleWebPath=/ajax/services/search/web
+googleLocalPath=/ajax/services/search/local
+googleVideoPath=/ajax/services/search/video
+googleBlogsPath=/ajax/services/search/blogs
+googleNewsPath=/ajax/services/search/news
+googleBookPath=/ajax/services/search/book
+googleImagePath=/ajax/services/search/image
+googlePatentPath=/ajax/services/search/patent
+
# This connection is restricted just for Schibsted S\u00F8k AS.
# It is a paid service so others must contact yahoo for their own details
(paid or demo) and override the following.
yahooIdpHost=schibsted-proxy.idp.inktomisearch.com
Modified: trunk/generic.sesam/war/src/main/conf/modes.xml
===================================================================
--- trunk/generic.sesam/war/src/main/conf/modes.xml 2009-09-30 12:34:54 UTC
(rev 7298)
+++ trunk/generic.sesam/war/src/main/conf/modes.xml 2009-09-30 14:43:54 UTC
(rev 7299)
@@ -89,6 +89,16 @@
</query-builder>
</esp-fast-command>
+ <google-command id="default-google-command" host="googleHost"
port="googlePort" results-to-return="8" inherit="default-command"/>
+ <google-command id="default-google-web-command" path="googleWebPath"
inherit="default-google-command"/>
+ <google-command id="default-google-local-command"
path="googleLocalPath" inherit="default-google-command"/>
+ <google-command id="default-google-video-command"
path="googleVideoPath" inherit="default-google-command"/>
+ <google-command id="default-google-blogs-command"
path="googleBlogsPath" inherit="default-google-command"/>
+ <google-command id="default-google-news-command" path="googleNewsPath"
inherit="default-google-command"/>
+ <google-command id="default-google-book-command" path="googleBookPath"
inherit="default-google-command"/>
+ <google-command id="default-google-image-command"
path="googleImagePath" inherit="default-google-command"/>
+ <google-command id="default-google-patent-command"
path="googlePatentPath" inherit="default-google-command"/>
+
<math-command id="default-math-command" inherit="default-command"/>
<!-- when overriding this command, remember to adjust partnerid
according to this:
_______________________________________________
Kernel-commits mailing list
[email protected]
http://sesat.no/mailman/listinfo/kernel-commits
