http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java b/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java index 9c7a0b2..cd3c399 100644 --- a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java +++ b/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java @@ -15,9 +15,11 @@ * limitations under the License. */ - package org.apache.zeppelin.interpreter.launcher; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterRunner; @@ -28,13 +30,7 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Interpreter Launcher which use shell script to launch the interpreter process. - */ +/** Interpreter Launcher which use shell script to launch the interpreter process. */ public class StandardInterpreterLauncher extends InterpreterLauncher { private static final Logger LOGGER = LoggerFactory.getLogger(StandardInterpreterLauncher.class); @@ -55,10 +51,7 @@ public class StandardInterpreterLauncher extends InterpreterLauncher { if (option.isExistingProcess()) { return new RemoteInterpreterRunningProcess( - context.getInterpreterSettingName(), - connectTimeout, - option.getHost(), - option.getPort()); + context.getInterpreterSettingName(), connectTimeout, option.getHost(), option.getPort()); } else { // try to recover it first if (zConf.isRecoveryEnabled()) { @@ -66,25 +59,38 @@ public class StandardInterpreterLauncher extends InterpreterLauncher { recoveryStorage.getInterpreterClient(context.getInterpreterGroupId()); if (recoveredClient != null) { if (recoveredClient.isRunning()) { - LOGGER.info("Recover interpreter process: " + recoveredClient.getHost() + ":" + - recoveredClient.getPort()); + LOGGER.info( + "Recover interpreter process: " + + recoveredClient.getHost() + + ":" + + recoveredClient.getPort()); return recoveredClient; } else { - LOGGER.warn("Cannot recover interpreter process: " + recoveredClient.getHost() + ":" - + recoveredClient.getPort() + ", as it is already terminated."); + LOGGER.warn( + "Cannot recover interpreter process: " + + recoveredClient.getHost() + + ":" + + recoveredClient.getPort() + + ", as it is already terminated."); } } } // create new remote process - String localRepoPath = zConf.getInterpreterLocalRepoPath() + "/" - + context.getInterpreterSettingId(); + String localRepoPath = + zConf.getInterpreterLocalRepoPath() + "/" + context.getInterpreterSettingId(); return new RemoteInterpreterManagedProcess( runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(), - context.getZeppelinServerRPCPort(), context.getZeppelinServerHost(), zConf.getInterpreterPortRange(), - zConf.getInterpreterDir() + "/" + groupName, localRepoPath, - buildEnvFromProperties(context), connectTimeout, name, - context.getInterpreterGroupId(), option.isUserImpersonate()); + context.getZeppelinServerRPCPort(), + context.getZeppelinServerHost(), + zConf.getInterpreterPortRange(), + zConf.getInterpreterDir() + "/" + groupName, + localRepoPath, + buildEnvFromProperties(context), + connectTimeout, + name, + context.getInterpreterGroupId(), + option.isUserImpersonate()); } }
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java b/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java index 1861636..4f05f29 100644 --- a/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java +++ b/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java @@ -17,18 +17,17 @@ package org.apache.zeppelin.interpreter.launcher; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Properties; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class StandardInterpreterLauncherTest { @Before public void setUp() { @@ -46,14 +45,26 @@ public class StandardInterpreterLauncherTest { properties.setProperty("property_1", "value_1"); InterpreterOption option = new InterpreterOption(); option.setUserImpersonate(true); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host"); + InterpreterLaunchContext context = + new InterpreterLaunchContext( + properties, + option, + null, + "user1", + "intpGroupId", + "groupId", + "groupName", + "name", + 0, + "host"); InterpreterClient client = launcher.launch(context); - assertTrue( client instanceof RemoteInterpreterManagedProcess); + assertTrue(client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; assertEquals("name", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir()); assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir()); - assertEquals(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getIntValue(), + assertEquals( + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getIntValue(), interpreterProcess.getConnectTimeout()); assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner()); assertEquals(1, interpreterProcess.getEnv().size()); @@ -70,9 +81,20 @@ public class StandardInterpreterLauncherTest { ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000"); InterpreterOption option = new InterpreterOption(); option.setUserImpersonate(true); - InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host"); + InterpreterLaunchContext context = + new InterpreterLaunchContext( + properties, + option, + null, + "user1", + "intpGroupId", + "groupId", + "groupName", + "name", + 0, + "host"); InterpreterClient client = launcher.launch(context); - assertTrue( client instanceof RemoteInterpreterManagedProcess); + assertTrue(client instanceof RemoteInterpreterManagedProcess); RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client; assertEquals("name", interpreterProcess.getInterpreterSettingName()); assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir()); @@ -82,5 +104,4 @@ public class StandardInterpreterLauncherTest { assertEquals(0, interpreterProcess.getEnv().size()); assertEquals(true, interpreterProcess.isUserImpersonated()); } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java index 12dbd90..63725cd 100644 --- a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java @@ -30,7 +30,6 @@ import java.io.InputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.net.URISyntaxException; -import java.security.InvalidKeyException; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -44,9 +43,7 @@ import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Azure storage backend for notebooks - */ +/** Azure storage backend for notebooks */ public class AzureNotebookRepo implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(AzureNotebookRepo.class); @@ -55,9 +52,7 @@ public class AzureNotebookRepo implements NotebookRepo { private String shareName; private CloudFileDirectory rootDir; - public AzureNotebookRepo() { - - } + public AzureNotebookRepo() {} public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; @@ -65,15 +60,18 @@ public class AzureNotebookRepo implements NotebookRepo { shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE); try { - CloudStorageAccount account = CloudStorageAccount.parse( - conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING)); + CloudStorageAccount account = + CloudStorageAccount.parse( + conf.getString( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING)); CloudFileClient client = account.createCloudFileClient(); CloudFileShare share = client.getShareReference(shareName); share.createIfNotExists(); - CloudFileDirectory userDir = StringUtils.isBlank(user) ? - share.getRootDirectoryReference() : - share.getRootDirectoryReference().getDirectoryReference(user); + CloudFileDirectory userDir = + StringUtils.isBlank(user) + ? share.getRootDirectoryReference() + : share.getRootDirectoryReference().getDirectoryReference(user); userDir.createIfNotExists(); rootDir = userDir.getDirectoryReference("notebook"); @@ -128,8 +126,8 @@ public class AzureNotebookRepo implements NotebookRepo { throw new IOException(msg, e); } - String json = IOUtils.toString(ins, - conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING)); + String json = + IOUtils.toString(ins, conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING)); ins.close(); return Note.fromJson(json); } @@ -199,8 +197,7 @@ public class AzureNotebookRepo implements NotebookRepo { } @Override - public void close() { - } + public void close() {} @Override public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { @@ -212,5 +209,4 @@ public class AzureNotebookRepo implements NotebookRepo { public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { LOG.warn("Method not implemented"); } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java index 5d9c85c..8d1bf4d 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java @@ -1,12 +1,10 @@ package org.apache.zeppelin.notebook.repo; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.FileSystemStorage; import org.apache.zeppelin.notebook.Note; @@ -15,25 +13,11 @@ import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - /** * NotebookRepos for hdfs. * - * Assume the notebook directory structure is as following - * - notebookdir - * - noteId/note.json - * - noteId/note.json - * - noteId/note.json + * <p>Assume the notebook directory structure is as following - notebookdir - noteId/note.json - + * noteId/note.json - noteId/note.json */ public class FileSystemNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class); @@ -41,14 +25,12 @@ public class FileSystemNotebookRepo implements NotebookRepo { private FileSystemStorage fs; private Path notebookDir; - public FileSystemNotebookRepo() { - - } + public FileSystemNotebookRepo() {} public void init(ZeppelinConfiguration zConf) throws IOException { this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir()); - LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() + - " for Zeppelin Notebook."); + LOGGER.info( + "Creating FileSystem: " + this.fs.getFs().getClass().getName() + " for Zeppelin Notebook."); this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir())); LOGGER.info("Using folder {} to store notebook", notebookDir); this.fs.tryMkDir(notebookDir); @@ -67,16 +49,15 @@ public class FileSystemNotebookRepo implements NotebookRepo { @Override public Note get(final String noteId, AuthenticationInfo subject) throws IOException { - String content = this.fs.readFile( - new Path(notebookDir.toString() + "/" + noteId + "/note.json")); + String content = + this.fs.readFile(new Path(notebookDir.toString() + "/" + noteId + "/note.json")); return Note.fromJson(content); } @Override public void save(final Note note, AuthenticationInfo subject) throws IOException { - this.fs.writeFile(note.toJson(), - new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), - true); + this.fs.writeFile( + note.toJson(), new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), true); } @Override @@ -99,5 +80,4 @@ public class FileSystemNotebookRepo implements NotebookRepo { public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { LOGGER.warn("updateSettings is not implemented for HdfsNotebookRepo"); } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java index 5fd8becc..cbbe5a6 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java @@ -1,6 +1,13 @@ package org.apache.zeppelin.notebook.repo; +import static org.junit.Assert.assertEquals; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -12,15 +19,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - public class FileSystemNotebookRepoTest { private ZeppelinConfiguration zConf; @@ -32,9 +30,11 @@ public class FileSystemNotebookRepoTest { @Before public void setUp() throws IOException { - notebookDir = Files.createTempDirectory("FileSystemNotebookRepoTest").toFile().getAbsolutePath(); + notebookDir = + Files.createTempDirectory("FileSystemNotebookRepoTest").toFile().getAbsolutePath(); zConf = new ZeppelinConfiguration(); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir); hadoopConf = new Configuration(); fs = FileSystem.get(hadoopConf); hdfsNotebookRepo = new FileSystemNotebookRepo(); @@ -80,7 +80,8 @@ public class FileSystemNotebookRepoTest { @Test public void testComplicatedScenarios() throws IOException { - // scenario_1: notebook_dir is not clean. There're some unrecognized dir and file under notebook_dir + // scenario_1: notebook_dir is not clean. There're some unrecognized dir and file under + // notebook_dir fs.mkdirs(new Path(notebookDir, "1/2")); OutputStream out = fs.create(new Path(notebookDir, "1/a.json")); out.close(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java index ad99aba..7647fbc 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -49,14 +49,14 @@ import org.slf4j.LoggerFactory; /** * A NotebookRepo implementation for storing notebooks in Google Cloud Storage. * - * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path - * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an - * object store, so this "directory" should not itself be an object. Instead, it represents the base - * path for the note.json files. + * <p>Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path must + * be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an object + * store, so this "directory" should not itself be an object. Instead, it represents the base path + * for the note.json files. * - * Authentication is provided by google-auth-library-java. - * @see <a href="https://github.com/google/google-auth-library-java";> - * google-auth-library-java</a>. + * <p>Authentication is provided by google-auth-library-java. + * + * @see <a href="https://github.com/google/google-auth-library-java";>google-auth-library-java</a>. */ public class GCSNotebookRepo implements NotebookRepo { @@ -67,8 +67,7 @@ public class GCSNotebookRepo implements NotebookRepo { private Pattern noteNamePattern; private Storage storage; - public GCSNotebookRepo() { - } + public GCSNotebookRepo() {} @VisibleForTesting public GCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException { @@ -78,37 +77,38 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void init(ZeppelinConfiguration zConf) throws IOException { - this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING); + this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING); String gcsStorageDir = zConf.getGCSStorageDir(); if (gcsStorageDir.isEmpty()) { throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'"); } if (!gcsStorageDir.startsWith("gs://")) { - throw new IOException(String.format( - "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir)); + throw new IOException( + String.format("GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir)); } String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length()); // pathComponents excludes empty string if trailing slash is present List<String> pathComponents = Arrays.asList(storageDirWithoutScheme.split("/")); if (pathComponents.size() < 1) { - throw new IOException(String.format( - "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir", - gcsStorageDir)); + throw new IOException( + String.format( + "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir", + gcsStorageDir)); } this.bucketName = pathComponents.get(0); if (pathComponents.size() > 1) { - this.basePath = Optional.of(StringUtils.join( - pathComponents.subList(1, pathComponents.size()), "/")); + this.basePath = + Optional.of(StringUtils.join(pathComponents.subList(1, pathComponents.size()), "/")); } else { this.basePath = Optional.absent(); } // Notes are stored at gs://bucketName/basePath/<note-id>/note.json if (basePath.isPresent()) { - this.noteNamePattern = Pattern.compile( - "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); + this.noteNamePattern = + Pattern.compile("^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); } else { this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$"); } @@ -130,13 +130,10 @@ public class GCSNotebookRepo implements NotebookRepo { List<NoteInfo> infos = new ArrayList<>(); Iterable<Blob> blobsUnderDir; if (basePath.isPresent()) { - blobsUnderDir = storage - .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/")) - .iterateAll(); + blobsUnderDir = + storage.list(bucketName, BlobListOption.prefix(this.basePath.get() + "/")).iterateAll(); } else { - blobsUnderDir = storage - .list(bucketName) - .iterateAll(); + blobsUnderDir = storage.list(bucketName).iterateAll(); } for (Blob b : blobsUnderDir) { Matcher matcher = noteNamePattern.matcher(b.getName()); @@ -172,9 +169,8 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void save(Note note, AuthenticationInfo subject) throws IOException { - BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) - .setContentType("application/json") - .build(); + BlobInfo info = + BlobInfo.newBuilder(makeBlobId(note.getId())).setContentType("application/json").build(); try { storage.create(info, note.toJson().getBytes("UTF-8")); } catch (StorageException se) { @@ -198,7 +194,7 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void close() { - //no-op + // no-op } @Override http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java index c1fae67..2c603ad 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java @@ -55,12 +55,13 @@ public class GCSNotebookRepoTest { @Parameters public static Collection<Object[]> data() { - return Arrays.asList(new Object[][] { - { "bucketname", Optional.absent(), "gs://bucketname" }, - { "bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/" }, - { "bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir" }, - { "bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/" } - }); + return Arrays.asList( + new Object[][] { + {"bucketname", Optional.absent(), "gs://bucketname"}, + {"bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/"}, + {"bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir"}, + {"bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/"} + }); } @Parameter(0) @@ -123,7 +124,8 @@ public class GCSNotebookRepoTest { try { notebookRepo.get("id", AUTH_INFO); fail(); - } catch (IOException e) {} + } catch (IOException e) { + } } @Test @@ -145,7 +147,8 @@ public class GCSNotebookRepoTest { try { notebookRepo.get("id", AUTH_INFO); fail(); - } catch (IOException e) {} + } catch (IOException e) { + } } @Test @@ -171,7 +174,8 @@ public class GCSNotebookRepoTest { try { notebookRepo.remove("id", AUTH_INFO); fail(); - } catch (IOException e) {} + } catch (IOException e) { + } } @Test @@ -200,16 +204,14 @@ public class GCSNotebookRepoTest { } private void create(Note note) throws IOException { - BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) - .setContentType("application/json") - .build(); + BlobInfo info = + BlobInfo.newBuilder(makeBlobId(note.getId())).setContentType("application/json").build(); storage.create(info, note.toJson().getBytes("UTF-8")); } private void createMalformed(String noteId) throws IOException { - BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId)) - .setContentType("application/json") - .build(); + BlobInfo info = + BlobInfo.newBuilder(makeBlobId(noteId)).setContentType("application/json").build(); storage.create(info, "{ invalid-json }".getBytes("UTF-8")); } @@ -221,7 +223,8 @@ public class GCSNotebookRepoTest { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), ""); new GCSNotebookRepo(new ZeppelinConfiguration(), storage); fail(); - } catch (IOException e) {} + } catch (IOException e) { + } } @Test @@ -230,6 +233,7 @@ public class GCSNotebookRepoTest { System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo"); new GCSNotebookRepo(new ZeppelinConfiguration(), storage); fail(); - } catch (IOException e) {} + } catch (IOException e) { + } } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java index 7729d52..29ed2f4 100644 --- a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -20,6 +20,10 @@ package org.apache.zeppelin.notebook.repo; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.Lists; +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.user.AuthenticationInfo; @@ -37,20 +41,14 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.List; - /** * NotebookRepo that hosts all the notebook FS in a single Git repo * - * This impl intended to be simple and straightforward: - * - does not handle branches - * - only basic local git file repo, no remote Github push\pull. GitHub integration is - * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * <p>This impl intended to be simple and straightforward: - does not handle branches - only basic + * local git file repo, no remote Github push\pull. GitHub integration is implemented in @see {@link + * org.apache.zeppelin.notebook.repo.GitNotebookRepo} * - * TODO(bzz): add default .gitignore + * <p>TODO(bzz): add default .gitignore */ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl { private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class); @@ -70,8 +68,8 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith @Override public void init(ZeppelinConfiguration conf) throws IOException { - //TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause - //AbstractMethodError + // TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause + // AbstractMethodError this.conf = conf; setNotebookDirectory(conf.getNotebookDir()); @@ -117,11 +115,8 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith } /** - * the idea is to: - * 1. stash current changes - * 2. remember head commit and checkout to the desired revision - * 3. get note and checkout back to the head - * 4. apply stash on top and remove it + * the idea is to: 1. stash current changes 2. remember head commit and checkout to the desired + * revision 3. get note and checkout back to the head 4. apply stash on top and remove it */ @Override public synchronized Note get(String noteId, String revId, AuthenticationInfo subject) @@ -149,7 +144,10 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith ObjectId applied = git.stashApply().setStashRef(stash.getName()).call(); ObjectId dropped = git.stashDrop().setStashRef(0).call(); Collection<RevCommit> stashes = git.stashList().call(); - LOG.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped, + LOG.debug( + "Stash applied as : {}, and dropped : {}, stash size: {}", + applied, + dropped, stashes.size()); } } catch (GitAPIException e) { @@ -164,12 +162,12 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith LOG.debug("Listing history for {}:", noteId); try { Iterable<RevCommit> logs = git.log().addPath(noteId).call(); - for (RevCommit log: logs) { + for (RevCommit log : logs) { history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime())); LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage()); } } catch (NoHeadException e) { - //when no initial commit exists + // when no initial commit exists LOG.warn("No Head found for {}, {}", noteId, e.getMessage()); } catch (GitAPIException e) { LOG.error("Failed to get logs for {}", noteId, e); @@ -186,13 +184,13 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith } return revisionNote; } - + @Override public void close() { git.getRepository().close(); } - //DI replacements for Tests + // DI replacements for Tests protected Git getGit() { return git; } @@ -200,6 +198,4 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith void setGit(Git git) { this.git = git; } - } - http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java index 58d7cc1..7de299d 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java @@ -20,11 +20,11 @@ package org.apache.zeppelin.notebook.repo; import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.mock; +import com.google.common.base.Joiner; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; - import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -44,8 +44,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Joiner; - public class GitNotebookRepoTest { private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepoTest.class); @@ -59,7 +57,8 @@ public class GitNotebookRepoTest { @Before public void setUp() throws Exception { - String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); + String zpath = + System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); zeppelinDir = new File(zpath); zeppelinDir.mkdirs(); new File(zeppelinDir, "conf").mkdirs(); @@ -71,24 +70,23 @@ public class GitNotebookRepoTest { String testNoteDir = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID); String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2); FileUtils.copyDirectory( - new File( - GitNotebookRepoTest.class.getResource( - Joiner.on(File.separator).join("", TEST_NOTE_ID) - ).getFile() - ), - new File(testNoteDir)); + new File( + GitNotebookRepoTest.class + .getResource(Joiner.on(File.separator).join("", TEST_NOTE_ID)) + .getFile()), + new File(testNoteDir)); FileUtils.copyDirectory( - new File( - GitNotebookRepoTest.class.getResource( - Joiner.on(File.separator).join("", TEST_NOTE_ID2) - ).getFile() - ), - new File(testNoteDir2) - ); + new File( + GitNotebookRepoTest.class + .getResource(Joiner.on(File.separator).join("", TEST_NOTE_ID2)) + .getFile()), + new File(testNoteDir2)); System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); conf = ZeppelinConfiguration.create(); } @@ -102,14 +100,14 @@ public class GitNotebookRepoTest { @Test public void initNonemptyNotebookDir() throws IOException, GitAPIException { - //given - .git does not exit + // given - .git does not exit File dotGit = new File(Joiner.on(File.separator).join(notebooksDir, ".git")); assertThat(dotGit.exists()).isEqualTo(false); - //when + // when notebookRepo = new GitNotebookRepo(conf); - //then + // then Git git = notebookRepo.getGit(); assertThat(git).isNotNull(); @@ -123,21 +121,21 @@ public class GitNotebookRepoTest { @Test public void showNotebookHistoryEmptyTest() throws GitAPIException, IOException { - //given + // given notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); - //when + // when List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, null); - //then - //no initial commit, empty history + // then + // no initial commit, empty history assertThat(testNotebookHistory).isEmpty(); } @Test public void showNotebookHistoryMultipleNotesTest() throws IOException { - //initial checks + // initial checks notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); @@ -145,13 +143,13 @@ public class GitNotebookRepoTest { assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null)).isEmpty(); - //add commit to both notes + // add commit to both notes notebookRepo.checkpoint(TEST_NOTE_ID, "first commit, note1", null); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); - //modify, save and checkpoint first note + // modify, save and checkpoint first note Note note = notebookRepo.get(TEST_NOTE_ID, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -164,10 +162,10 @@ public class GitNotebookRepoTest { assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null)) - .isEqualTo(Revision.EMPTY); + .isEqualTo(Revision.EMPTY); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); - //modify, save and checkpoint second note + // modify, save and checkpoint second note note = notebookRepo.get(TEST_NOTE_ID2, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -213,7 +211,7 @@ public class GitNotebookRepoTest { } private boolean containsNote(List<NoteInfo> notes, String noteId) { - for (NoteInfo note: notes) { + for (NoteInfo note : notes) { if (note.getId().equals(noteId)) { return true; } @@ -323,7 +321,7 @@ public class GitNotebookRepoTest { @Test public void setRevisionTest() throws IOException { - //create repo and check that note doesn't contain revisions + // create repo and check that note doesn't contain revisions notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); @@ -337,7 +335,7 @@ public class GitNotebookRepoTest { // checkpoint revision1 Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: first commit", null); - //TODO(khalid): change to EMPTY after rebase + // TODO(khalid): change to EMPTY after rebase assertThat(revision1).isNotNull(); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); @@ -354,7 +352,7 @@ public class GitNotebookRepoTest { // checkpoint revision2 Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: second commit", null); - //TODO(khalid): change to EMPTY after rebase + // TODO(khalid): change to EMPTY after rebase assertThat(revision2).isNotNull(); assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java index 738d708..bf35f67 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java @@ -17,6 +17,11 @@ package org.apache.zeppelin.notebook.repo; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock; @@ -26,33 +31,31 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -//TODO(zjffdu) move it to zeppelin-zengine +// TODO(zjffdu) move it to zeppelin-zengine public class NotebookRepoSyncInitializationTest { - private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class); + private static final Logger LOG = + LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class); private String validFirstStorageClass = "org.apache.zeppelin.notebook.repo.VFSNotebookRepo"; - private String validSecondStorageClass = "org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"; + private String validSecondStorageClass = + "org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"; private String invalidStorageClass = "org.apache.zeppelin.notebook.repo.DummyNotebookRepo"; private String validOneStorageConf = validFirstStorageClass; private String validTwoStorageConf = validFirstStorageClass + "," + validSecondStorageClass; private String invalidTwoStorageConf = validFirstStorageClass + "," + invalidStorageClass; - private String unsupportedStorageConf = validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass; + private String unsupportedStorageConf = + validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass; private String emptyStorageConf = ""; @Before - public void setUp(){ - System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); - //setup routine + public void setUp() { + System.setProperty( + ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); + // setup routine } @After public void tearDown() { - //tear-down routine + // tear-down routine } @Test @@ -71,11 +74,12 @@ public class NotebookRepoSyncInitializationTest { @Test public void validInitTwoStorageTest() throws IOException { // initialize folders for each storage - String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + String zpath = + System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); File mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath+"/notebook"; + String mainNotePath = zpath + "/notebook"; String secNotePath = mainNotePath + "_secondary"; File mainNotebookDir = new File(mainNotePath); File secNotebookDir = new File(secNotePath); @@ -84,7 +88,8 @@ public class NotebookRepoSyncInitializationTest { // set confs System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validTwoStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo @@ -111,11 +116,12 @@ public class NotebookRepoSyncInitializationTest { @Test public void initUnsupportedNumberStoragesTest() throws IOException { // initialize folders for each storage, currently for 2 only - String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); + String zpath = + System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); File mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath+"/notebook"; + String mainNotePath = zpath + "/notebook"; String secNotePath = mainNotePath + "_secondary"; File mainNotebookDir = new File(mainNotePath); File secNotebookDir = new File(secNotePath); @@ -124,7 +130,8 @@ public class NotebookRepoSyncInitializationTest { // set confs System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), unsupportedStorageConf); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo @@ -149,7 +156,7 @@ public class NotebookRepoSyncInitializationTest { @Test public void initOneDummyStorageTest() throws IOException { - // set confs + // set confs System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), invalidStorageClass); ZeppelinConfiguration conf = ZeppelinConfiguration.create(); // create repo @@ -158,4 +165,4 @@ public class NotebookRepoSyncInitializationTest { assertEquals(notebookRepoSync.getRepoCount(), 1); assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepo); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 68e2615..a012f72 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -17,6 +17,17 @@ package org.apache.zeppelin.notebook.repo; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -32,7 +43,6 @@ import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.ParagraphJobListener; -import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; @@ -46,20 +56,7 @@ import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - - -//TODO(zjffdu) move it to zeppelin-zengine +// TODO(zjffdu) move it to zeppelin-zengine public class NotebookRepoSyncTest implements ParagraphJobListener { private File mainZepDir; @@ -80,11 +77,12 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { @Before public void setUp() throws Exception { - String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis(); + String zpath = + System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis(); mainZepDir = new File(zpath); mainZepDir.mkdirs(); new File(mainZepDir, "conf").mkdirs(); - String mainNotePath = zpath+"/notebook"; + String mainNotePath = zpath + "/notebook"; String secNotePath = mainNotePath + "_secondary"; mainNotebookDir = new File(mainNotePath); secNotebookDir = new File(secNotePath); @@ -92,11 +90,16 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { secNotebookDir.mkdirs(); System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock"); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false"); - System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf"); - System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); + System.setProperty( + ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf"); + System.setProperty( + ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath()); LOG.info("main Note dir : " + mainNotePath); LOG.info("secondary note dir : " + secNotePath); @@ -107,16 +110,29 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { this.schedulerFactory = SchedulerFactory.singleton(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - interpreterSettingManager = new InterpreterSettingManager(conf, - mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); + interpreterSettingManager = + new InterpreterSettingManager( + conf, + mock(AngularObjectRegistryListener.class), + mock(RemoteInterpreterProcessListener.class), + mock(ApplicationEventListener.class)); factory = new InterpreterFactory(interpreterSettingManager); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); notebookAuthorization = NotebookAuthorization.init(conf); credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath(), null); - notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, - notebookAuthorization, credentials); + notebookSync = + new Notebook( + conf, + notebookRepoSync, + schedulerFactory, + factory, + interpreterSettingManager, + this, + search, + notebookAuthorization, + credentials); anonymous = new AuthenticationInfo("anonymous"); } @@ -143,7 +159,9 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { // check that automatically saved on both storages assertEquals(1, notebookRepoSync.list(0, anonymous).size()); assertEquals(1, notebookRepoSync.list(1, anonymous).size()); - assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); + assertEquals( + notebookRepoSync.list(0, anonymous).get(0).getId(), + notebookRepoSync.list(1, anonymous).get(0).getId()); notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous); } @@ -160,7 +178,9 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { /* check that created in both storage systems */ assertEquals(1, notebookRepoSync.list(0, anonymous).size()); assertEquals(1, notebookRepoSync.list(1, anonymous).size()); - assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId()); + assertEquals( + notebookRepoSync.list(0, anonymous).get(0).getId(), + notebookRepoSync.list(1, anonymous).get(0).getId()); /* remove Note */ notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous); @@ -168,7 +188,6 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { /* check that deleted in both storages */ assertEquals(0, notebookRepoSync.list(0, anonymous).size()); assertEquals(0, notebookRepoSync.list(1, anonymous).size()); - } @Test @@ -186,30 +205,58 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { assertEquals(1, note.getParagraphs().size()); /* new paragraph not yet saved into storages */ - assertEquals(0, notebookRepoSync.get(0, - notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); - assertEquals(0, notebookRepoSync.get(1, - notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + assertEquals( + 0, + notebookRepoSync + .get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous) + .getParagraphs() + .size()); + assertEquals( + 0, + notebookRepoSync + .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) + .getParagraphs() + .size()); /* save to storage under index 0 (first storage) */ notebookRepoSync.save(0, note, anonymous); /* check paragraph saved to first storage */ - assertEquals(1, notebookRepoSync.get(0, - notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + assertEquals( + 1, + notebookRepoSync + .get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous) + .getParagraphs() + .size()); /* check paragraph isn't saved to second storage */ - assertEquals(0, notebookRepoSync.get(1, - notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + assertEquals( + 0, + notebookRepoSync + .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) + .getParagraphs() + .size()); /* apply sync */ notebookRepoSync.sync(null); /* check whether added to second storage */ - assertEquals(1, notebookRepoSync.get(1, - notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size()); + assertEquals( + 1, + notebookRepoSync + .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) + .getParagraphs() + .size()); /* check whether same paragraph id */ - assertEquals(p1.getId(), notebookRepoSync.get(0, - notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getLastParagraph().getId()); - assertEquals(p1.getId(), notebookRepoSync.get(1, - notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getLastParagraph().getId()); + assertEquals( + p1.getId(), + notebookRepoSync + .get(0, notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous) + .getLastParagraph() + .getId()); + assertEquals( + p1.getId(), + notebookRepoSync + .get(1, notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous) + .getLastParagraph() + .getId()); notebookRepoSync.remove(note.getId(), anonymous); } @@ -240,12 +287,22 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { @Test public void testOneWaySyncOnReloadedList() throws IOException, SchedulerException { - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true"); conf = ZeppelinConfiguration.create(); notebookRepoSync = new NotebookRepoSync(conf); - notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, - notebookAuthorization, credentials); + notebookSync = + new Notebook( + conf, + notebookRepoSync, + schedulerFactory, + factory, + interpreterSettingManager, + this, + search, + notebookAuthorization, + credentials); // check that both storage repos are empty assertTrue(notebookRepoSync.getRepoCount() > 1); @@ -288,12 +345,23 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { @Test public void testCheckpointOneStorage() throws IOException, SchedulerException { - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); + System.setProperty( + ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.GitNotebookRepo"); ZeppelinConfiguration vConf = ZeppelinConfiguration.create(); NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf); - Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search, - notebookAuthorization, credentials); + Notebook vNotebookSync = + new Notebook( + vConf, + vRepoSync, + schedulerFactory, + factory, + interpreterSettingManager, + this, + search, + notebookAuthorization, + credentials); // one git versioned storage initialized assertThat(vRepoSync.getRepoCount()).isEqualTo(1); @@ -354,18 +422,30 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { notebookRepoSync.save(1, note, null); /* check paragraph isn't saved into first storage */ - assertEquals(0, notebookRepoSync.get(0, - notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); + assertEquals( + 0, + notebookRepoSync + .get(0, notebookRepoSync.list(0, null).get(0).getId(), null) + .getParagraphs() + .size()); /* check paragraph is saved into second storage */ - assertEquals(1, notebookRepoSync.get(1, - notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size()); + assertEquals( + 1, + notebookRepoSync + .get(1, notebookRepoSync.list(1, null).get(0).getId(), null) + .getParagraphs() + .size()); /* now sync by user1 */ notebookRepoSync.sync(user1); /* check that note updated and acl are same on main storage*/ - assertEquals(1, notebookRepoSync.get(0, - notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size()); + assertEquals( + 1, + notebookRepoSync + .get(0, notebookRepoSync.list(0, null).get(0).getId(), null) + .getParagraphs() + .size()); assertEquals(true, authInfo.isOwner(note.getId(), entity)); assertEquals(1, authInfo.getOwners(note.getId()).size()); assertEquals(0, authInfo.getReaders(note.getId()).size()); @@ -397,44 +477,31 @@ public class NotebookRepoSyncTest implements ParagraphJobListener { assertEquals(true, authInfo.isWriter(note.getId(), entity)); } - static void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ - delete(f); - } + static void delete(File file) { + if (file.isFile()) file.delete(); + else if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null && files.length > 0) { + for (File f : files) { + delete(f); } - file.delete(); } + file.delete(); + } } - - @Override - public void onOutputAppend(Paragraph paragraph, int idx, String output) { - - } + public void onOutputAppend(Paragraph paragraph, int idx, String output) {} @Override - public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage msg) { - - } + public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage msg) {} @Override - public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) { - - } + public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) {} @Override - public void onProgressUpdate(Paragraph paragraph, int progress) { - } + public void onProgressUpdate(Paragraph paragraph, int progress) {} @Override - public void onStatusChange(Paragraph paragraph, Status before, Status after) { - - } - - + public void onStatusChange(Paragraph paragraph, Status before, Status after) {} } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java index 7f450df..4d8bf52 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java @@ -16,12 +16,11 @@ */ package org.apache.zeppelin.notebook.repo.mock; +import java.io.IOException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; -import java.io.IOException; - public class VFSNotebookRepoMock extends VFSNotebookRepo { public VFSNotebookRepoMock() { @@ -32,5 +31,4 @@ public class VFSNotebookRepoMock extends VFSNotebookRepo { public void init(ZeppelinConfiguration conf) throws IOException { super.init(conf); } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java index 7d0415b..19da9d5 100644 --- a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.notebook.repo; +import java.io.IOException; +import java.net.URISyntaxException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.user.AuthenticationInfo; import org.eclipse.jgit.api.Git; @@ -29,22 +31,18 @@ import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.net.URISyntaxException; - /** - * GitHub integration to store notebooks in a GitHub repository. - * It uses the same simple logic implemented in @see - * {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} + * GitHub integration to store notebooks in a GitHub repository. It uses the same simple logic + * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo} * - * The logic for updating the local repository from the remote repository is the following: - * - When the <code>GitHubNotebookRepo</code> is initialized - * - When pushing the changes to the remote repository + * <p>The logic for updating the local repository from the remote repository is the following: - + * When the <code>GitHubNotebookRepo</code> is initialized - When pushing the changes to the remote + * repository * - * The logic for updating the remote repository on GitHub from local repository is the following: + * <p>The logic for updating the remote repository on GitHub from local repository is the following: * - When commit the changes (saving the notebook) * - * You should be able to use this integration with all remote git repositories that accept + * <p>You should be able to use this integration with all remote git repositories that accept * username + password authentication, not just GitHub. */ public class GitHubNotebookRepo extends GitNotebookRepo { @@ -98,11 +96,9 @@ public class GitHubNotebookRepo extends GitNotebookRepo { LOG.debug("Pulling latest changes from remote stream"); PullCommand pullCommand = git.pull(); pullCommand.setCredentialsProvider( - new UsernamePasswordCredentialsProvider( - zeppelinConfiguration.getZeppelinNotebookGitUsername(), - zeppelinConfiguration.getZeppelinNotebookGitAccessToken() - ) - ); + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken())); pullCommand.call(); @@ -116,11 +112,9 @@ public class GitHubNotebookRepo extends GitNotebookRepo { LOG.debug("Pushing latest changes to remote stream"); PushCommand pushCommand = git.push(); pushCommand.setCredentialsProvider( - new UsernamePasswordCredentialsProvider( - zeppelinConfiguration.getZeppelinNotebookGitUsername(), - zeppelinConfiguration.getZeppelinNotebookGitAccessToken() - ) - ); + new UsernamePasswordCredentialsProvider( + zeppelinConfiguration.getZeppelinNotebookGitUsername(), + zeppelinConfiguration.getZeppelinNotebookGitAccessToken())); pushCommand.call(); } catch (GitAPIException e) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java index e331714..de59e94 100644 --- a/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java @@ -1,4 +1,3 @@ - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -18,8 +17,12 @@ package org.apache.zeppelin.notebook.repo; +import static org.mockito.Mockito.mock; import com.google.common.base.Joiner; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.interpreter.InterpreterFactory; @@ -37,17 +40,11 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.Iterator; - -import static org.mockito.Mockito.mock; - /** - * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories created locally - * to handle the tracking of Git actions (pushes and pulls). The repositories are: - * 1. The first repository is considered as a remote that mimics a remote GitHub directory - * 2. The second repository is considered as the local notebook repository + * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories + * created locally to handle the tracking of Git actions (pushes and pulls). The repositories are: + * 1. The first repository is considered as a remote that mimics a remote GitHub directory 2. The + * second repository is considered as the local notebook repository */ public class GitHubNotebookRepoTest { private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepoTest.class); @@ -67,10 +64,10 @@ public class GitHubNotebookRepoTest { public void setUp() throws Exception { conf = ZeppelinConfiguration.create(); - String remoteRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" + - System.currentTimeMillis(); - String localRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + - System.currentTimeMillis(); + String remoteRepositoryPath = + System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" + System.currentTimeMillis(); + String localRepositoryPath = + System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis(); // Create a fake remote notebook Git repository locally in another directory remoteZeppelinDir = new File(remoteRepositoryPath); @@ -87,37 +84,46 @@ public class GitHubNotebookRepoTest { File notebookDir = new File(localNotebooksDir); notebookDir.mkdirs(); - // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote Git directory + // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote + // Git directory String remoteTestNoteDir = Joiner.on(File.separator).join(remoteNotebooksDir, TEST_NOTE_ID); FileUtils.copyDirectory( - new File( - GitHubNotebookRepoTest.class.getResource( - Joiner.on(File.separator).join("", TEST_NOTE_ID) - ).getFile() - ), new File(remoteTestNoteDir) - ); + new File( + GitHubNotebookRepoTest.class + .getResource(Joiner.on(File.separator).join("", TEST_NOTE_ID)) + .getFile()), + new File(remoteTestNoteDir)); // Create the fake remote Git repository - Repository remoteRepository = new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git")); + Repository remoteRepository = + new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git")); remoteRepository.create(); remoteGit = new Git(remoteRepository); remoteGit.add().addFilepattern(".").call(); - firstCommitRevision = remoteGit.commit().setMessage("First commit from remote repository").call(); + firstCommitRevision = + remoteGit.commit().setMessage("First commit from remote repository").call(); // Set the Git and Git configurations - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), remoteZeppelinDir.getAbsolutePath()); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), + remoteZeppelinDir.getAbsolutePath()); + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), + notebookDir.getAbsolutePath()); // Set the GitHub configurations System.setProperty( - ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), - "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo"); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(), - remoteNotebooksDir + File.separator + ".git"); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token"); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(), - "access-token"); + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), + "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo"); + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(), + remoteNotebooksDir + File.separator + ".git"); + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token"); + System.setProperty( + ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(), + "access-token"); // Create the Notebook repository (configured for the local repository) gitHubNotebookRepo = new GitHubNotebookRepo(); @@ -127,9 +133,9 @@ public class GitHubNotebookRepoTest { @After public void tearDown() throws Exception { // Cleanup the temporary folders uses as Git repositories - File[] temporaryFolders = { remoteZeppelinDir, localZeppelinDir }; + File[] temporaryFolders = {remoteZeppelinDir, localZeppelinDir}; - for(File temporaryFolder : temporaryFolders) { + for (File temporaryFolder : temporaryFolders) { if (!FileUtils.deleteQuietly(temporaryFolder)) LOG.error("Failed to delete {} ", temporaryFolder.getName()); } @@ -137,66 +143,75 @@ public class GitHubNotebookRepoTest { @Test /** - * Test the case when the Notebook repository is created, it pulls the latest changes from the remote repository + * Test the case when the Notebook repository is created, it pulls the latest changes from the + * remote repository */ - public void pullChangesFromRemoteRepositoryOnLoadingNotebook() throws IOException, GitAPIException { - NotebookRepoWithVersionControl.Revision firstHistoryRevision = gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0); + public void pullChangesFromRemoteRepositoryOnLoadingNotebook() + throws IOException, GitAPIException { + NotebookRepoWithVersionControl.Revision firstHistoryRevision = + gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0); - assert(this.firstCommitRevision.getName().equals(firstHistoryRevision.id)); + assert (this.firstCommitRevision.getName().equals(firstHistoryRevision.id)); } @Test /** - * Test the case when the check-pointing (add new files and commit) it also pulls the latest changes from the - * remote repository + * Test the case when the check-pointing (add new files and commit) it also pulls the latest + * changes from the remote repository */ public void pullChangesFromRemoteRepositoryOnCheckpointing() throws GitAPIException, IOException { // Create a new commit in the remote repository - RevCommit secondCommitRevision = remoteGit.commit().setMessage("Second commit from remote repository").call(); + RevCommit secondCommitRevision = + remoteGit.commit().setMessage("Second commit from remote repository").call(); // Add a new paragraph to the local repository addParagraphToNotebook(TEST_NOTE_ID); // Commit and push the changes to remote repository - NotebookRepoWithVersionControl.Revision thirdCommitRevision = gitHubNotebookRepo.checkpoint( - TEST_NOTE_ID, "Third commit from local repository", null); + NotebookRepoWithVersionControl.Revision thirdCommitRevision = + gitHubNotebookRepo.checkpoint(TEST_NOTE_ID, "Third commit from local repository", null); - // Check all the commits as seen from the local repository. The commits are ordered chronologically. The last + // Check all the commits as seen from the local repository. The commits are ordered + // chronologically. The last // commit is the first in the commit logs. Iterator<RevCommit> revisions = gitHubNotebookRepo.getGit().log().all().call().iterator(); revisions.next(); // The Merge `master` commit after pushing to the remote repository - assert(thirdCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph + assert (thirdCommitRevision.id.equals( + revisions.next().getName())); // The local commit after adding the paragraph // The second commit done on the remote repository - assert(secondCommitRevision.getName().equals(revisions.next().getName())); + assert (secondCommitRevision.getName().equals(revisions.next().getName())); // The first commit done on the remote repository - assert(firstCommitRevision.getName().equals(revisions.next().getName())); + assert (firstCommitRevision.getName().equals(revisions.next().getName())); } @Test /** - * Test the case when the check-pointing (add new files and commit) it pushes the local commits to the remote - * repository + * Test the case when the check-pointing (add new files and commit) it pushes the local commits to + * the remote repository */ - public void pushLocalChangesToRemoteRepositoryOnCheckpointing() throws IOException, GitAPIException { + public void pushLocalChangesToRemoteRepositoryOnCheckpointing() + throws IOException, GitAPIException { // Add a new paragraph to the local repository addParagraphToNotebook(TEST_NOTE_ID); // Commit and push the changes to remote repository - NotebookRepoWithVersionControl.Revision secondCommitRevision = gitHubNotebookRepo.checkpoint( - TEST_NOTE_ID, "Second commit from local repository", null); + NotebookRepoWithVersionControl.Revision secondCommitRevision = + gitHubNotebookRepo.checkpoint(TEST_NOTE_ID, "Second commit from local repository", null); - // Check all the commits as seen from the remote repository. The commits are ordered chronologically. The last + // Check all the commits as seen from the remote repository. The commits are ordered + // chronologically. The last // commit is the first in the commit logs. Iterator<RevCommit> revisions = remoteGit.log().all().call().iterator(); - assert(secondCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph + assert (secondCommitRevision.id.equals( + revisions.next().getName())); // The local commit after adding the paragraph // The first commit done on the remote repository - assert(firstCommitRevision.getName().equals(revisions.next().getName())); + assert (firstCommitRevision.getName().equals(revisions.next().getName())); } private void addParagraphToNotebook(String noteId) throws IOException { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java index 618568d..0bc85fd 100644 --- a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java @@ -30,9 +30,7 @@ import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Backend for storing Notebook on MongoDB - */ +/** Backend for storing Notebook on MongoDB */ public class MongoNotebookRepo implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class); @@ -41,9 +39,7 @@ public class MongoNotebookRepo implements NotebookRepo { private MongoDatabase db; private MongoCollection<Document> coll; - public MongoNotebookRepo() { - - } + public MongoNotebookRepo() {} public void init(ZeppelinConfiguration conf) throws IOException { this.conf = conf; @@ -59,15 +55,14 @@ public class MongoNotebookRepo implements NotebookRepo { } /** - * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true, - * this method will insert local notes into MongoDB on startup. - * If a note already exists in MongoDB, skip it. + * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true, this method will insert + * local notes into MongoDB on startup. If a note already exists in MongoDB, skip it. */ private void insertFileSystemNotes() throws IOException { LinkedList<Document> docs = new LinkedList<>(); // docs to be imported NotebookRepo vfsRepo = new VFSNotebookRepo(); vfsRepo.init(conf); - List<NoteInfo> infos = vfsRepo.list(null); + List<NoteInfo> infos = vfsRepo.list(null); // collect notes to be imported for (NoteInfo info : infos) { Note note = vfsRepo.get(info.getId(), null); @@ -83,15 +78,16 @@ public class MongoNotebookRepo implements NotebookRepo { try { coll.insertMany(docs, new InsertManyOptions().ordered(false)); } catch (MongoBulkWriteException e) { - printDuplicatedException(e); //print duplicated document warning log + printDuplicatedException(e); // print duplicated document warning log } - vfsRepo.close(); // it does nothing for now but maybe in the future... + vfsRepo.close(); // it does nothing for now but maybe in the future... } /** - * MongoBulkWriteException contains error messages that inform - * which documents were duplicated. This method catches those ID and print them. + * MongoBulkWriteException contains error messages that inform which documents were duplicated. + * This method catches those ID and print them. + * * @param e */ private void printDuplicatedException(MongoBulkWriteException e) { @@ -127,19 +123,17 @@ public class MongoNotebookRepo implements NotebookRepo { } /** - * Find documents of which type of _id is object ID, and change it to note ID. - * Since updating _id field is not allowed, remove original documents and insert - * new ones with string _id(note ID) + * Find documents of which type of _id is object ID, and change it to note ID. Since updating _id + * field is not allowed, remove original documents and insert new ones with string _id(note ID) */ private void syncId() { // find documents whose id type is object id - MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator(); + MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator(); // if there is no such document, exit - if (!cursor.hasNext()) - return; + if (!cursor.hasNext()) return; - List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update - List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted + List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update + List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted while (cursor.hasNext()) { Document doc = cursor.next(); @@ -158,9 +152,7 @@ public class MongoNotebookRepo implements NotebookRepo { cursor.close(); } - /** - * Convert document to note - */ + /** Convert document to note */ private Note documentToNote(Document doc) { // document to JSON String json = doc.toJson(); @@ -168,9 +160,7 @@ public class MongoNotebookRepo implements NotebookRepo { return Note.fromJson(json); } - /** - * Convert note to document - */ + /** Convert note to document */ private Document noteToDocument(Note note) { // note to JSON String json = note.toJson(); @@ -218,5 +208,4 @@ public class MongoNotebookRepo implements NotebookRepo { public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { LOG.warn("Method not implemented"); } - }
