Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java?rev=1488537&r1=1488536&r2=1488537&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
(original)
+++
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
Sat Jun 1 15:30:15 2013
@@ -1,125 +1,181 @@
-/*
- * To change this template, choose Tools | Templates
- * and open the template in the editor.
- */
-package org.apache.manifoldcf.crawler.connectors.googledrive;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.googleapis.media.MediaHttpDownloader;
-import com.google.api.client.http.GenericUrl;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.drive.Drive;
-import java.util.Map;
-
-
-
-import java.util.HashMap;
-import java.util.HashSet;
-import com.google.api.services.drive.model.File;
-import com.google.api.services.drive.model.FileList;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- *
- * @author andrew
- */
-public class GoogleDriveSession {
-
- private static String APPNAME = "Searchbox Google drive Connector";
- private Drive drive;
- private static HttpTransport HTTP_TRANSPORT;
- private static final JsonFactory JSON_FACTORY = new JacksonFactory();
-
- GoogleDriveSession(Map<String, String> parameters) {
- try {
-
- HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport();
- GoogleCredential credentials = new
GoogleCredential.Builder().setClientSecrets(parameters.get("clientid"),
parameters.get("clientsecret"))
-
.setJsonFactory(JSON_FACTORY).setTransport(HTTP_TRANSPORT).build().setRefreshToken(parameters.get("refreshtoken"));
-
- drive = new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY,
credentials).setApplicationName(APPNAME).build();
- } catch (Exception ex) {
-
Logger.getLogger(GoogleDriveSession.class.getName()).log(Level.SEVERE, null,
ex);
- }
- }
-
- public Map<String, String> getRepositoryInfo() throws IOException {
- Map<String, String> info = new HashMap<String, String>();
- info.put("Application Name", drive.getApplicationName());
- info.put("Base URL", drive.getBaseUrl());
- return info;
- }
-
- HashSet<String> getSeeds(String googleDriveQuery) throws IOException {
- HashSet<String> ids = new HashSet<String>();
- Drive.Files.List request;
-
- request = drive.files().list().setQ(googleDriveQuery);
-
- do {
- try {
- FileList files = request.execute();
- for (File f : files.getItems()) {
- ids.add(f.getId());
- }
- request.setPageToken(files.getNextPageToken());
- } catch (IOException e) {
- System.out.println("An error occurred: " + e);
- request.setPageToken(null);
- }
- } while (request.getPageToken() != null
- && request.getPageToken().length() > 0);
- return ids;
- }
-
- public File getObject(String id) throws IOException {
- File file = drive.files().get(id).execute();
- return file;
- }
-
- List<String> getChildren(String nodeId) throws IOException {
- ArrayList<String> ids = new ArrayList<String>();
- Drive.Files.List request = drive.files().list().setQ("'" + nodeId + "'
in parents");
-
- do {
- try {
- FileList files = request.execute();
- for (File f : files.getItems()) {
- ids.add(f.getId());
- }
- request.setPageToken(files.getNextPageToken());
- } catch (IOException e) {
-
- request.setPageToken(null);
- }
- } while (request.getPageToken() != null
- && request.getPageToken().length() > 0);
- return ids;
- }
-
- String getUrl(File googleFile, String exportType) {
- if (googleFile.containsKey("fileSize")) {
- return googleFile.getDownloadUrl();
- } else {
- return googleFile.getExportLinks().get(exportType);
- }
- }
-
- ByteArrayOutputStream getGoogleDriveOutputStream(String documentURI)
throws IOException {
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-
- MediaHttpDownloader downloader =
- new MediaHttpDownloader(HTTP_TRANSPORT,
drive.getRequestFactory().getInitializer());
- downloader.setDirectDownloadEnabled(false);
- downloader.download(new GenericUrl(documentURI), outputStream);
- return outputStream;
- }
-}
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.manifoldcf.crawler.connectors.googledrive;
+
+import org.apache.manifoldcf.core.common.*;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.googleapis.media.MediaHttpDownloader;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.drive.Drive;
+import java.util.Map;
+
+
+
+import java.util.HashMap;
+import java.util.HashSet;
+import com.google.api.services.drive.model.File;
+import com.google.api.services.drive.model.FileList;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import java.security.GeneralSecurityException;
+
+/**
+ *
+ * @author andrew
+ */
+public class GoogleDriveSession {
+
+ private static String APPNAME = "Searchbox Google drive Connector";
+
+ private Drive drive;
+ private HttpTransport HTTP_TRANSPORT;
+
+ private static final JsonFactory JSON_FACTORY = new JacksonFactory();
+
+ /** Constructor. Create a session.
+ */
+ public GoogleDriveSession(Map<String, String> parameters)
+ throws IOException, GeneralSecurityException {
+ HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport();
+
+ GoogleCredential credentials = new
GoogleCredential.Builder().setClientSecrets(parameters.get("clientid"),
parameters.get("clientsecret"))
+
.setJsonFactory(JSON_FACTORY).setTransport(HTTP_TRANSPORT).build().setRefreshToken(parameters.get("refreshtoken"));
+
+ drive = new Drive.Builder(HTTP_TRANSPORT, JSON_FACTORY,
credentials).setApplicationName(APPNAME).build();
+ }
+
+ /** Obtain repository information.
+ */
+ public Map<String, String> getRepositoryInfo() throws IOException {
+ Map<String, String> info = new HashMap<String, String>();
+ info.put("Application Name", drive.getApplicationName());
+ info.put("Base URL", drive.getBaseUrl());
+ return info;
+ }
+
+ /** Get the list of matching root documents, e.g. seeds.
+ */
+ public void getSeeds(XThreadStringBuffer idBuffer, String googleDriveQuery)
+ throws IOException, InterruptedException {
+ Drive.Files.List request;
+
+ request = drive.files().list().setQ(googleDriveQuery);
+
+ do {
+ FileList files = request.execute();
+ for (File f : files.getItems()) {
+ idBuffer.add(f.getId());
+ }
+ request.setPageToken(files.getNextPageToken());
+ } while (request.getPageToken() != null
+ && request.getPageToken().length() > 0);
+ }
+
+ /** Get an individual document.
+ */
+ public File getObject(String id) throws IOException {
+ File file = drive.files().get(id).execute();
+ return file;
+ }
+
+ /** Get the list of child documents for a document.
+ */
+ public void getChildren(XThreadStringBuffer idBuffer, String nodeId)
+ throws IOException, InterruptedException {
+ Drive.Files.List request = drive.files().list().setQ("'" + nodeId + "' in
parents");
+
+ do {
+ FileList files = request.execute();
+ for (File f : files.getItems()) {
+ idBuffer.add(f.getId());
+ }
+ request.setPageToken(files.getNextPageToken());
+ } while (request.getPageToken() != null
+ && request.getPageToken().length() > 0);
+ }
+
+
+ /** Get a stream representing the specified document.
+ */
+ public void getGoogleDriveOutputStream(XThreadInputStream inputStream,
String documentURI) throws IOException {
+ // Create an object that implements outputstream but pushes everything
through to the designated input stream
+ OutputStream outputStream = new XThreadOutputStream(inputStream);
+ try {
+ MediaHttpDownloader downloader =
+ new MediaHttpDownloader(HTTP_TRANSPORT,
drive.getRequestFactory().getInitializer());
+ downloader.setDirectDownloadEnabled(false);
+ downloader.download(new GenericUrl(documentURI), outputStream);
+ } finally {
+ // Make sure it is closed and flushed
+ outputStream.close();
+ }
+ }
+
+ protected static class XThreadOutputStream extends OutputStream {
+ protected final XThreadInputStream inputStream;
+
+ byte[] byteBuffer = new byte[1];
+
+ public XThreadOutputStream(XThreadInputStream inputStream) {
+ this.inputStream = inputStream;
+ }
+
+ @Override
+ public void write(int c)
+ throws IOException {
+ byteBuffer[0] = (byte)c;
+ try {
+ inputStream.stuffQueue(byteBuffer,0,1);
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void write(byte[] buffer, int pos, int amt)
+ throws IOException {
+ try {
+ inputStream.stuffQueue(buffer,pos,amt);
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void close()
+ throws IOException {
+ inputStream.doneStuffingQueue();
+ super.close();
+ }
+
+ // MHL
+ }
+}
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveSession.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/Messages.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/Messages.java?rev=1488537&r1=1488536&r2=1488537&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/Messages.java
(original)
+++
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/Messages.java
Sat Jun 1 15:30:15 2013
@@ -1,141 +1,141 @@
-/* $Id: Messages.java 1295926 2012-03-01 21:56:27Z kwright $ */
-
-/**
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.manifoldcf.crawler.connectors.googledrive;
-
-import java.util.Locale;
-import java.util.Map;
-import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
-import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
-
-public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
-{
- public static final String
DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.crawler.connectors.googledrive.common";
- public static final String
DEFAULT_PATH_NAME="org.apache.manifoldcf.crawler.connectors.googledrive";
-
- /** Constructor - do no instantiate
- */
- protected Messages()
- {
- }
-
- public static String getString(Locale locale, String messageKey)
- {
- return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
- }
-
- public static String getAttributeString(Locale locale, String messageKey)
- {
- return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
- }
-
- public static String getBodyString(Locale locale, String messageKey)
- {
- return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
- }
-
- public static String getAttributeJavascriptString(Locale locale, String
messageKey)
- {
- return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, null);
- }
-
- public static String getBodyJavascriptString(Locale locale, String
messageKey)
- {
- return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
null);
- }
-
- public static String getString(Locale locale, String messageKey, Object[]
args)
- {
- return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
- }
-
- public static String getAttributeString(Locale locale, String messageKey,
Object[] args)
- {
- return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
- }
-
- public static String getBodyString(Locale locale, String messageKey,
Object[] args)
- {
- return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
- }
-
- public static String getAttributeJavascriptString(Locale locale, String
messageKey, Object[] args)
- {
- return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, args);
- }
-
- public static String getBodyJavascriptString(Locale locale, String
messageKey, Object[] args)
- {
- return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
args);
- }
-
- // More general methods which allow bundlenames and class loaders to be
specified.
-
- public static String getString(String bundleName, Locale locale, String
messageKey, Object[] args)
- {
- return getString(Messages.class, bundleName, locale, messageKey, args);
- }
-
- public static String getAttributeString(String bundleName, Locale locale,
String messageKey, Object[] args)
- {
- return getAttributeString(Messages.class, bundleName, locale, messageKey,
args);
- }
-
- public static String getBodyString(String bundleName, Locale locale, String
messageKey, Object[] args)
- {
- return getBodyString(Messages.class, bundleName, locale, messageKey, args);
- }
-
- public static String getAttributeJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args)
- {
- return getAttributeJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
- }
-
- public static String getBodyJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args)
- {
- return getBodyJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
- }
-
- // Resource output
-
- public static void outputResource(IHTTPOutput output, Locale locale, String
resourceKey,
- Map<String,String> substitutionParameters, boolean mapToUpperCase)
- throws ManifoldCFException
- {
- outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
- substitutionParameters,mapToUpperCase);
- }
-
- public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
- Map<String,String> substitutionParameters, boolean mapToUpperCase)
- throws ManifoldCFException
- {
-
outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
- substitutionParameters,mapToUpperCase);
- }
-
- public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
- Map<String,Object> contextObjects)
- throws ManifoldCFException
- {
-
outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
- contextObjects);
- }
-
-}
-
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.googledrive;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+ public static final String
DEFAULT_BUNDLE_NAME="org.apache.manifoldcf.crawler.connectors.googledrive.common";
+ public static final String
DEFAULT_PATH_NAME="org.apache.manifoldcf.crawler.connectors.googledrive";
+
+ /** Constructor - do no instantiate
+ */
+ protected Messages()
+ {
+ }
+
+ public static String getString(Locale locale, String messageKey)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String
messageKey)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, null);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String
messageKey)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
null);
+ }
+
+ public static String getString(Locale locale, String messageKey, Object[]
args)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey,
Object[] args)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey,
Object[] args)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String
messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale,
messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String
messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey,
args);
+ }
+
+ // More general methods which allow bundlenames and class loaders to be
specified.
+
+ public static String getString(String bundleName, Locale locale, String
messageKey, Object[] args)
+ {
+ return getString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(String bundleName, Locale locale,
String messageKey, Object[] args)
+ {
+ return getAttributeString(Messages.class, bundleName, locale, messageKey,
args);
+ }
+
+ public static String getBodyString(String bundleName, Locale locale, String
messageKey, Object[] args)
+ {
+ return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(String bundleName, Locale
locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(Messages.class, bundleName, locale,
messageKey, args);
+ }
+
+ // Resource output
+
+ public static void outputResource(IHTTPOutput output, Locale locale, String
resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+
outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale
locale, String resourceKey,
+ Map<String,Object> contextObjects)
+ throws ManifoldCFException
+ {
+
outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ contextObjects);
+ }
+
+}
+
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/Messages.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/Messages.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_en_US.properties?rev=1488537&r1=1488536&r2=1488537&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_en_US.properties
(original)
+++
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_en_US.properties
Sat Jun 1 15:30:15 2013
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
GoogleDriveRepositoryConnector.Server=Server
Added:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_ja_JP.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_ja_JP.properties?rev=1488537&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_ja_JP.properties
(added)
+++
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_ja_JP.properties
Sat Jun 1 15:30:15 2013
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GoogleDriveRepositoryConnector.Server=Server
+
+
+GoogleDriveRepositoryConnector.RefreshToken=RefreshToken:
+GoogleDriveRepositoryConnector.ClientID=Client ID:
+GoogleDriveRepositoryConnector.ClientSecret=Client Secret ID:
+
+
+GoogleDriveRepositoryConnector.RefreshTokenMustNotBeNull=Refresh Token must
not be null
+GoogleDriveRepositoryConnector.ClientSecretMustNotBeNull=Client Secret must
not be null
+GoogleDriveRepositoryConnector.ClientMustNotBeNull=Client must not be null
+
+
+GoogleDriveRepositoryConnector.GoogleDriveQuery=Google Drive Seed Query
+GoogleDriveRepositoryConnector.GoogleDriveQueryColon=Google Drive Seed Query:
+
+GoogleDriveRepositoryConnector.ParametersColon=Parameters:
+GoogleDriveRepositoryConnector.UsernameEquals=username=
+GoogleDriveRepositoryConnector.PasswordEquals=password=
+
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_ja_JP.properties
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/branches/CONNECTORS-694/connectors/googledrive/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/googledrive/common_ja_JP.properties
------------------------------------------------------------------------------
svn:keywords = Id
Modified:
manifoldcf/branches/CONNECTORS-694/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1488537&r1=1488536&r2=1488537&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
(original)
+++
manifoldcf/branches/CONNECTORS-694/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
Sat Jun 1 15:30:15 2013
@@ -2145,7 +2145,7 @@ public class WikiConnector extends org.a
try
{
HttpRequestBase executeMethod =
getInitializedGetMethod(getListPagesURL(startPageTitle,namespace,prefix));
- PageBuffer pageBuffer = new PageBuffer();
+ XThreadStringBuffer pageBuffer = new XThreadStringBuffer();
ExecuteListPagesThread t = new
ExecuteListPagesThread(httpClient,executeMethod,pageBuffer,startPageTitle);
try
{
@@ -2275,12 +2275,12 @@ public class WikiConnector extends org.a
protected HttpClient client;
protected HttpRequestBase executeMethod;
protected Throwable exception = null;
- protected PageBuffer pageBuffer;
+ protected XThreadStringBuffer pageBuffer;
protected String lastPageTitle = null;
protected String startPageTitle;
protected boolean loginNeeded = false;
- public ExecuteListPagesThread(HttpClient client, HttpRequestBase
executeMethod, PageBuffer pageBuffer, String startPageTitle)
+ public ExecuteListPagesThread(HttpClient client, HttpRequestBase
executeMethod, XThreadStringBuffer pageBuffer, String startPageTitle)
{
super();
setDaemon(true);
@@ -2361,7 +2361,7 @@ public class WikiConnector extends org.a
* </query-continue>
* </api>
*/
- protected static boolean parseListPagesResponse(InputStream is, PageBuffer
buffer, String startPageTitle, ReturnString lastTitle)
+ protected static boolean parseListPagesResponse(InputStream is,
XThreadStringBuffer buffer, String startPageTitle, ReturnString lastTitle)
throws ManifoldCFException, ServiceInterruption
{
// Parse the document. This will cause various things to occur, within
the instantiated XMLContext class.
@@ -2393,11 +2393,11 @@ public class WikiConnector extends org.a
protected static class WikiListPagesAPIContext extends SingleLevelContext
{
protected String lastTitle = null;
- protected PageBuffer buffer;
+ protected XThreadStringBuffer buffer;
protected String startPageTitle;
protected boolean loginNeeded = false;
- public WikiListPagesAPIContext(XMLStream theStream, PageBuffer buffer,
String startPageTitle)
+ public WikiListPagesAPIContext(XMLStream theStream, XThreadStringBuffer
buffer, String startPageTitle)
{
super(theStream,"api");
this.buffer = buffer;
@@ -2434,11 +2434,11 @@ public class WikiConnector extends org.a
protected static class WikiListPagesQueryContext extends
SingleLevelErrorContext
{
protected String lastTitle = null;
- protected PageBuffer buffer;
+ protected XThreadStringBuffer buffer;
protected String startPageTitle;
public WikiListPagesQueryContext(XMLStream theStream, String namespaceURI,
String localName, String qName, Attributes atts,
- PageBuffer buffer, String startPageTitle)
+ XThreadStringBuffer buffer, String startPageTitle)
{
super(theStream,namespaceURI,localName,qName,atts,"query");
this.buffer = buffer;
@@ -2469,11 +2469,11 @@ public class WikiConnector extends org.a
protected static class WikiListPagesAllPagesContext extends
SingleLevelContext
{
protected String lastTitle = null;
- protected PageBuffer buffer;
+ protected XThreadStringBuffer buffer;
protected String startPageTitle;
public WikiListPagesAllPagesContext(XMLStream theStream, String
namespaceURI, String localName, String qName, Attributes atts,
- PageBuffer buffer, String startPageTitle)
+ XThreadStringBuffer buffer, String startPageTitle)
{
super(theStream,namespaceURI,localName,qName,atts,"allpages");
this.buffer = buffer;
@@ -2506,11 +2506,11 @@ public class WikiConnector extends org.a
protected static class WikiListPagesPContext extends BaseProcessingContext
{
protected String lastTitle = null;
- protected PageBuffer buffer;
+ protected XThreadStringBuffer buffer;
protected String startPageTitle;
public WikiListPagesPContext(XMLStream theStream, String namespaceURI,
String localName, String qName, Attributes atts,
- PageBuffer buffer, String startPageTitle)
+ XThreadStringBuffer buffer, String startPageTitle)
{
super(theStream,namespaceURI,localName,qName,atts);
this.buffer = buffer;
Modified:
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadInputStream.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadInputStream.java?rev=1488537&r1=1488536&r2=1488537&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadInputStream.java
(original)
+++
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadInputStream.java
Sat Jun 1 15:30:15 2013
@@ -26,20 +26,27 @@ import java.io.*;
*/
public class XThreadInputStream extends InputStream
{
- private byte[] buffer = new byte[65536];
+ private final byte[] buffer = new byte[65536];
private int startPoint = 0;
private int byteCount = 0;
private boolean streamEnd = false;
private IOException failureException = null;
- private InputStream sourceStream;
private boolean abort = false;
-
- /** Constructor */
+
+ private final InputStream sourceStream;
+
+ /** Constructor, from a given input stream. */
public XThreadInputStream(InputStream sourceStream)
{
this.sourceStream = sourceStream;
}
+ /** Constructor, from another source. */
+ public XThreadInputStream()
+ {
+ this.sourceStream = null;
+ }
+
/** Call this method to abort the stuffQueue() method.
*/
public void abort()
@@ -51,8 +58,64 @@ public class XThreadInputStream extends
}
}
+ /** This method is called from the helper thread side, to stuff bytes onto
+ * the queue when there is no input stream.
+ * It exits only when interrupted or done.
+ */
+ public void stuffQueue(byte[] byteBuffer, int offset, int amount)
+ throws InterruptedException
+ {
+ while (amount > 0)
+ {
+ int maxToRead;
+ int readStartPoint;
+ synchronized (this)
+ {
+ if (abort || streamEnd)
+ return;
+ // Calculate amount to read
+ maxToRead = buffer.length - byteCount;
+ if (maxToRead == 0)
+ {
+ wait();
+ continue;
+ }
+ readStartPoint = (startPoint + byteCount) & (buffer.length-1);
+ }
+ if (readStartPoint + maxToRead >= buffer.length)
+ maxToRead = buffer.length - readStartPoint;
+ // Now, copy to buffer
+ int amt;
+ if (amount > maxToRead)
+ amt = maxToRead;
+ else
+ amt = amount;
+ //??? make sure this is source -> target
+ System.arraycopy(byteBuffer,offset,buffer,readStartPoint,amt);
+ offset += amt;
+ amount -= amt;
+ synchronized (this)
+ {
+ byteCount += amt;
+ notifyAll();
+ }
+ }
+ }
+
+ /** Call this method when there is no more data to write.
+ */
+ public void doneStuffingQueue()
+ {
+ synchronized (this)
+ {
+ streamEnd = true;
+ notifyAll();
+ }
+ }
+
/** This method is called from the helper thread side, to keep the queue
- * stuffed. It exits when the stream is empty, or when interrupted.
+ * stuffed from the input stream.
+ * It exits when the stream is empty, or when interrupted.
*/
public void stuffQueue()
throws IOException, InterruptedException
Added:
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadStringBuffer.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadStringBuffer.java?rev=1488537&view=auto
==============================================================================
---
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadStringBuffer.java
(added)
+++
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadStringBuffer.java
Sat Jun 1 15:30:15 2013
@@ -0,0 +1,89 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.common;
+
+import java.util.*;
+
+/** Thread-safe class that functions as a limited-size buffer of strings */
+public class XThreadStringBuffer
+{
+ protected static int MAX_SIZE = 1024;
+
+ protected List<String> buffer = new ArrayList<String>(MAX_SIZE);
+
+ protected boolean complete = false;
+ protected boolean abandoned = false;
+
+ /** Constructor */
+ public XThreadStringBuffer()
+ {
+ }
+
+ /** Add a string to the buffer, and block if the buffer is full */
+ public synchronized void add(String string)
+ throws InterruptedException
+ {
+ while (buffer.size() == MAX_SIZE && !abandoned)
+ wait();
+ if (abandoned)
+ return;
+ buffer.add(string);
+ // Notify threads that are waiting on there being stuff in the queue
+ notifyAll();
+ }
+
+ /** Signal that the buffer should be abandoned.
+ * Called by the receiving thread! */
+ public synchronized void abandon()
+ {
+ abandoned = true;
+ // Notify waiting threads
+ notifyAll();
+ }
+
+ /** Signal that the operation is complete, and that no more strings
+ * will be added. Called by the sending thread!
+ */
+ public synchronized void signalDone()
+ {
+ complete = true;
+ // Notify threads that are waiting for stuff to appear, because it won't
+ notifyAll();
+ }
+
+ /** Pull an id off the buffer, and wait if there's more to come.
+ * Called by the receiving thread!
+ * Returns null if the operation is complete.
+ */
+ public synchronized String fetch()
+ throws InterruptedException
+ {
+ while (buffer.size() == 0 && !complete)
+ wait();
+ if (buffer.size() == 0)
+ return null;
+ boolean isBufferFull = (buffer.size() == MAX_SIZE);
+ String rval = buffer.remove(buffer.size()-1);
+ // Notify those threads waiting on buffer being not completely full to wake
+ if (isBufferFull)
+ notifyAll();
+ return rval;
+ }
+
+}
Propchange:
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadStringBuffer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
manifoldcf/branches/CONNECTORS-694/framework/core/src/main/java/org/apache/manifoldcf/core/common/XThreadStringBuffer.java
------------------------------------------------------------------------------
svn:keywords = Id
