http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/notebook/Zeppelin Tutorial/Using Pig for querying data_2C57UKYWR.zpln ---------------------------------------------------------------------- diff --git a/notebook/Zeppelin Tutorial/Using Pig for querying data_2C57UKYWR.zpln b/notebook/Zeppelin Tutorial/Using Pig for querying data_2C57UKYWR.zpln new file mode 100644 index 0000000..04cb008 --- /dev/null +++ b/notebook/Zeppelin Tutorial/Using Pig for querying data_2C57UKYWR.zpln @@ -0,0 +1,334 @@ +{ + "paragraphs": [ + { + "text": "%md\n\n\n### [Apache Pig](http://pig.apache.org/) is a platform for analyzing large data sets that consists of a high-level language for expressing data analysis programs, coupled with infrastructure for evaluating these programs. The salient property of Pig programs is that their structure is amenable to substantial parallelization, which in turns enables them to handle very large data sets.\n\nPig\u0027s language layer currently consists of a textual language called Pig Latin, which has the following key properties:\n\n* Ease of programming. It is trivial to achieve parallel execution of simple, \"embarrassingly parallel\" data analysis tasks. Complex tasks comprised of multiple interrelated data transformations are explicitly encoded as data flow sequences, making them easy to write, understand, and maintain.\n* Optimization opportunities. The way in which tasks are encoded permits the system to optimize their execution automatically, allowing the user to focus on semantics rather than efficiency.\n* Extensibility. Users can create their own functions to do special-purpose processing.\n", + "user": "anonymous", + "dateUpdated": "Jan 22, 2017 12:48:50 PM", + "config": { + "colWidth": 12.0, + "enabled": true, + "results": {}, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "editorHide": true, + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch3\u003e\u003ca href\u003d\"http://pig.apache.org/\"\u003eApache Pig\u003c/a\u003e is a platform for analyzing large data sets that consists of a high-level language for expressing data analysis programs, coupled with infrastructure for evaluating these programs. The salient property of Pig programs is that their structure is amenable to substantial parallelization, which in turns enables them to handle very large data sets.\u003c/h3\u003e\n\u003cp\u003ePig\u0026rsquo;s language layer currently consists of a textual language called Pig Latin, which has the following key properties:\u003c/p\u003e\n\u003cul\u003e\n \u003cli\u003eEase of programming. It is trivial to achieve parallel execution of simple, \u0026ldquo;embarrassingly parallel\u0026rdquo; data analysis tasks. Complex tasks comprised of multiple interrelated data transformations are explicitly encoded as data flow sequences, making them easy to write, understand, and maintain.\u003c/li\u003e\n \u003cli\u003eOptimization opportunities. The way in which tasks are encoded permits the system to optimize their execution automatically, allowing the user to focus on semantics rather than efficiency.\u003c/li\u003e\n \u003cli\u003eExtensibility. Users can create their own functions to do special-purpose processing.\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1483277502513_1156234051", + "id": "20170101-213142_1565013608", + "dateCreated": "Jan 1, 2017 9:31:42 PM", + "dateStarted": "Jan 22, 2017 12:48:50 PM", + "dateFinished": "Jan 22, 2017 12:48:51 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%md\n\nThis pig tutorial use pig to do the same thing as spark tutorial. The default mode is mapreduce, you can also use other modes like local/tez_local/tez. For mapreduce mode, you need to have hadoop installed and export `HADOOP_CONF_DIR` in `zeppelin-env.sh`\n\nThe tutorial consists of 3 steps.\n\n* Use shell interpreter to download bank.csv and upload it to hdfs\n* use `%pig` to process the data\n* use `%pig.query` to query the data", + "user": "anonymous", + "dateUpdated": "Jan 22, 2017 12:48:55 PM", + "config": { + "colWidth": 12.0, + "enabled": true, + "results": {}, + "editorSetting": { + "language": "markdown", + "editOnDblClick": true + }, + "editorMode": "ace/mode/markdown", + "editorHide": true, + "tableHide": false + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "HTML", + "data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003cp\u003eThis pig tutorial use pig to do the same thing as spark tutorial. The default mode is mapreduce, you can also use other modes like local/tez_local/tez. For mapreduce mode, you need to have hadoop installed and export \u003ccode\u003eHADOOP_CONF_DIR\u003c/code\u003e in \u003ccode\u003ezeppelin-env.sh\u003c/code\u003e\u003c/p\u003e\n\u003cp\u003eThe tutorial consists of 3 steps.\u003c/p\u003e\n\u003cul\u003e\n \u003cli\u003eUse shell interpreter to download bank.csv and upload it to hdfs\u003c/li\u003e\n \u003cli\u003euse \u003ccode\u003e%pig\u003c/code\u003e to process the data\u003c/li\u003e\n \u003cli\u003euse \u003ccode\u003e%pig.query\u003c/code\u003e to query the data\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/div\u003e" + } + ] + }, + "apps": [], + "jobName": "paragraph_1483689316217_-629483391", + "id": "20170106-155516_1050601059", + "dateCreated": "Jan 6, 2017 3:55:16 PM", + "dateStarted": "Jan 22, 2017 12:48:55 PM", + "dateFinished": "Jan 22, 2017 12:48:55 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%sh\n\nwget https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\nhadoop fs -put bank.csv .\n", + "user": "anonymous", + "dateUpdated": "Jan 22, 2017 12:51:48 PM", + "config": { + "colWidth": 12.0, + "enabled": true, + "results": {}, + "editorSetting": { + "language": "text", + "editOnDblClick": false + }, + "editorMode": "ace/mode/text" + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TEXT", + "data": "--2017-01-22 12:51:48-- https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\nResolving s3.amazonaws.com... 52.216.80.227\nConnecting to s3.amazonaws.com|52.216.80.227|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 461474 (451K) [application/octet-stream]\nSaving to: \u0027bank.csv.3\u0027\n\n 0K .......... .......... .......... .......... .......... 11% 141K 3s\n 50K .......... .......... .......... .......... .......... 22% 243K 2s\n 100K .......... .......... .......... .......... .......... 33% 449K 1s\n 150K .......... .......... .......... .......... .......... 44% 413K 1s\n 200K .......... .......... .......... .......... .......... 55% 746K 1s\n 250K .......... .......... .......... .......... .......... 66% 588K 0s\n 300K .......... .......... .......... .......... .......... 77% 840K 0s\n 350K .......... .......... .......... .......... .......... 88% 795K 0s\n 400K .......... ...... .... .......... .......... .......... 99% 1.35M 0s\n 450K 100% 13.2K\u003d1.1s\n\n2017-01-22 12:51:50 (409 KB/s) - \u0027bank.csv.3\u0027 saved [461474/461474]\n\n17/01/22 12:51:51 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1485058437578_-1906301827", + "id": "20170122-121357_640055590", + "dateCreated": "Jan 22, 2017 12:13:57 PM", + "dateStarted": "Jan 22, 2017 12:51:48 PM", + "dateFinished": "Jan 22, 2017 12:51:52 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%pig\n\nbankText \u003d load \u0027bank.csv\u0027 using PigStorage(\u0027;\u0027);\nbank \u003d foreach bankText generate $0 as age, $1 as job, $2 as marital, $3 as education, $5 as balance; \nbank \u003d filter bank by age !\u003d \u0027\"age\"\u0027;\nbank \u003d foreach bank generate (int)age, REPLACE(job,\u0027\"\u0027,\u0027\u0027) as job, REPLACE(marital, \u0027\"\u0027, \u0027\u0027) as marital, (int)(REPLACE(balance, \u0027\"\u0027, \u0027\u0027)) as balance;\n\n-- The following statement is optional, it depends on whether your needs.\n-- store bank into \u0027clean_bank.csv\u0027 using PigStorage(\u0027;\u0027);\n\n\n", + "user": "anonymous", + "dateUpdated": "Feb 24, 2017 5:08:08 PM", + "config": { + "colWidth": 12.0, + "editorMode": "ace/mode/pig", + "results": {}, + "enabled": true, + "editorSetting": { + "language": "pig", + "editOnDblClick": false + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [] + }, + "apps": [], + "jobName": "paragraph_1483277250237_-466604517", + "id": "20161228-140640_1560978333", + "dateCreated": "Jan 1, 2017 9:27:30 PM", + "dateStarted": "Feb 24, 2017 5:08:08 PM", + "dateFinished": "Feb 24, 2017 5:08:11 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%pig.query\n\nbank_data \u003d filter bank by age \u003c 30;\nb \u003d group bank_data by age;\nforeach b generate group, COUNT($1);\n\n", + "user": "anonymous", + "dateUpdated": "Feb 24, 2017 5:08:13 PM", + "config": { + "colWidth": 4.0, + "editorMode": "ace/mode/pig", + "results": { + "0": { + "graph": { + "mode": "multiBarChart", + "height": 300.0, + "optionOpen": false + }, + "helium": {} + } + }, + "enabled": true, + "editorSetting": { + "language": "pig", + "editOnDblClick": false + } + }, + "settings": { + "params": {}, + "forms": {} + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "group\tcol_1\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1483277250238_-465450270", + "id": "20161228-140730_1903342877", + "dateCreated": "Jan 1, 2017 9:27:30 PM", + "dateStarted": "Feb 24, 2017 5:08:13 PM", + "dateFinished": "Feb 24, 2017 5:08:26 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%pig.query\n\nbank_data \u003d filter bank by age \u003c ${maxAge\u003d40};\nb \u003d group bank_data by age;\nforeach b generate group, COUNT($1) as count;", + "user": "anonymous", + "dateUpdated": "Feb 24, 2017 5:08:14 PM", + "config": { + "colWidth": 4.0, + "editorMode": "ace/mode/pig", + "results": { + "0": { + "graph": { + "mode": "pieChart", + "height": 300.0, + "optionOpen": false + }, + "helium": {} + } + }, + "enabled": true, + "editorSetting": { + "language": "pig", + "editOnDblClick": false + } + }, + "settings": { + "params": { + "maxAge": "36" + }, + "forms": { + "maxAge": { + "name": "maxAge", + "defaultValue": "40", + "hidden": false + } + } + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "group\tcount\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n35\t180\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1483277250239_-465835019", + "id": "20161228-154918_1551591203", + "dateCreated": "Jan 1, 2017 9:27:30 PM", + "dateStarted": "Feb 24, 2017 5:08:14 PM", + "dateFinished": "Feb 24, 2017 5:08:29 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%pig.query\n\nbank_data \u003d filter bank by marital\u003d\u003d\u0027${marital\u003dsingle,single|divorced|married}\u0027;\nb \u003d group bank_data by age;\nforeach b generate group, COUNT($1) as count;\n\n\n", + "user": "anonymous", + "dateUpdated": "Feb 24, 2017 5:08:15 PM", + "config": { + "colWidth": 4.0, + "editorMode": "ace/mode/pig", + "results": { + "0": { + "graph": { + "mode": "scatterChart", + "height": 300.0, + "optionOpen": false + }, + "helium": {} + } + }, + "enabled": true, + "editorSetting": { + "language": "pig", + "editOnDblClick": false + } + }, + "settings": { + "params": { + "marital": "married" + }, + "forms": { + "marital": { + "name": "marital", + "defaultValue": "single", + "options": [ + { + "value": "single" + }, + { + "value": "divorced" + }, + { + "value": "married" + } + ], + "hidden": false + } + } + }, + "results": { + "code": "SUCCESS", + "msg": [ + { + "type": "TABLE", + "data": "group\tcount\n23\t3\n24\t11\n25\t11\n26\t18\n27\t26\n28\t23\n29\t37\n30\t56\n31\t104\n32\t105\n33\t103\n34\t142\n35\t109\n36\t117\n37\t100\n38\t99\n39\t88\n40\t105\n41\t97\n42\t91\n43\t79\n44\t68\n45\t76\n46\t82\n47\t78\n48\t91\n49\t87\n50\t74\n51\t63\n52\t66\n53\t75\n54\t56\n55\t68\n56\t50\n57\t78\n58\t67\n59\t56\n60\t36\n61\t15\n62\t5\n63\t7\n64\t6\n65\t4\n66\t7\n67\t5\n68\t1\n69\t5\n70\t5\n71\t5\n72\t4\n73\t6\n74\t2\n75\t3\n76\t1\n77\t5\n78\t2\n79\t3\n80\t6\n81\t1\n83\t2\n86\t1\n87\t1\n" + } + ] + }, + "apps": [], + "jobName": "paragraph_1483277250240_-480070728", + "id": "20161228-142259_575675591", + "dateCreated": "Jan 1, 2017 9:27:30 PM", + "dateStarted": "Feb 24, 2017 5:08:27 PM", + "dateFinished": "Feb 24, 2017 5:08:31 PM", + "status": "FINISHED", + "progressUpdateIntervalMs": 500 + }, + { + "text": "%pig\n", + "dateUpdated": "Jan 1, 2017 9:27:30 PM", + "config": {}, + "settings": { + "params": {}, + "forms": {} + }, + "apps": [], + "jobName": "paragraph_1483277250240_-480070728", + "id": "20161228-155036_1854903164", + "dateCreated": "Jan 1, 2017 9:27:30 PM", + "status": "READY", + "errorMessage": "", + "progressUpdateIntervalMs": 500 + } + ], + "name": "Using Pig for querying data", + "id": "2C57UKYWR", + "angularObjects": { + "2C3RWCVAG:shared_process": [], + "2C9KGCHDE:shared_process": [], + "2C8X2BS16:shared_process": [] + }, + "config": {}, + "info": {} +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java index a4d16ee..ab8cd66 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java @@ -110,5 +110,5 @@ public class SchedulerFactory { public ExecutorService getExecutor() { return executor; } - + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java index 12dbd90..e1e10e6 100644 --- a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java @@ -24,15 +24,12 @@ import com.microsoft.azure.storage.file.CloudFileClient; import com.microsoft.azure.storage.file.CloudFileDirectory; import com.microsoft.azure.storage.file.CloudFileShare; import com.microsoft.azure.storage.file.ListFileItem; -import java.io.ByteArrayOutputStream; + import java.io.IOException; import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; import java.net.URISyntaxException; -import java.security.InvalidKeyException; import java.util.Collections; -import java.util.LinkedList; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -48,7 +45,7 @@ import org.slf4j.LoggerFactory; * Azure storage backend for notebooks */ public class AzureNotebookRepo implements NotebookRepo { - private static final Logger LOG = LoggerFactory.getLogger(AzureNotebookRepo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AzureNotebookRepo.class); private ZeppelinConfiguration conf; private String user; @@ -84,50 +81,46 @@ public class AzureNotebookRepo implements NotebookRepo { } @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - List<NoteInfo> infos = new LinkedList<>(); - NoteInfo info = null; + public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException { + return list(rootDir); + } + private Map<String, NoteInfo> list(CloudFileDirectory folder) throws IOException { + Map<String, NoteInfo> notesInfo = new HashMap<>(); for (ListFileItem item : rootDir.listFilesAndDirectories()) { - if (item.getClass() == CloudFileDirectory.class) { + if (item instanceof CloudFileDirectory) { CloudFileDirectory dir = (CloudFileDirectory) item; - - try { - if (dir.getFileReference("note.json").exists()) { - info = new NoteInfo(getNote(dir.getName())); - - if (info != null) { - infos.add(info); - } + notesInfo.putAll(list(dir)); + } else if (item instanceof CloudFile){ + CloudFile file = (CloudFile) item; + if (file.getName().endsWith(".zpln")) { + try { + String noteName = getNotePath(rootDir.getUri().getPath(), file.getUri().getPath()); + String noteId = getNoteId(file.getUri().getPath()); + notesInfo.put(noteId, new NoteInfo(noteId, noteName)); + } catch (IOException e) { + LOGGER.warn(e.getMessage()); } - } catch (StorageException | URISyntaxException e) { - String msg = "Error enumerating notebooks from Azure storage"; - LOG.error(msg, e); - } catch (Exception e) { - LOG.error(e.getMessage(), e); + } else { + LOGGER.debug("Skip invalid note file: " + file.getUri().getPath()); } } } - - return infos; + return notesInfo; } - private Note getNote(String noteId) throws IOException { + @Override + public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException { InputStream ins = null; - try { - CloudFileDirectory dir = rootDir.getDirectoryReference(noteId); - CloudFile file = dir.getFileReference("note.json"); - - ins = file.openRead(); + CloudFile noteFile = rootDir.getFileReference(buildNoteFileName(noteId, notePath)); + ins = noteFile.openRead(); } catch (URISyntaxException | StorageException e) { - String msg = String.format("Error reading notebook %s from Azure storage", noteId); - - LOG.error(msg, e); - + String msg = String.format("Error reading notebook %s from Azure storage", + buildNoteFileName(noteId, notePath)); + LOGGER.error(msg, e); throw new IOException(msg, e); } - String json = IOUtils.toString(ins, conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING)); ins.close(); @@ -135,82 +128,60 @@ public class AzureNotebookRepo implements NotebookRepo { } @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { - return getNote(noteId); - } - - @Override public void save(Note note, AuthenticationInfo subject) throws IOException { - String json = note.toJson(); - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - Writer writer = new OutputStreamWriter(output); - writer.write(json); - writer.close(); - output.close(); - - byte[] buffer = output.toByteArray(); - try { - CloudFileDirectory dir = rootDir.getDirectoryReference(note.getId()); - dir.createIfNotExists(); - - CloudFile cloudFile = dir.getFileReference("note.json"); - cloudFile.uploadFromByteArray(buffer, 0, buffer.length); + CloudFile noteFile = rootDir.getFileReference(buildNoteFileName(note)); + noteFile.getParent().createIfNotExists(); + noteFile.uploadText(note.toJson()); } catch (URISyntaxException | StorageException e) { - String msg = String.format("Error saving notebook %s to Azure storage", note.getId()); - - LOG.error(msg, e); - + String msg = String.format("Error saving notebook %s to Azure storage", + buildNoteFileName(note)); + LOGGER.error(msg, e); throw new IOException(msg, e); } } - // unfortunately, we need to use a recursive delete here - private void delete(ListFileItem item) throws StorageException { - if (item.getClass() == CloudFileDirectory.class) { - CloudFileDirectory dir = (CloudFileDirectory) item; + @Override + public void move(String noteId, String notePath, String newNotePath, AuthenticationInfo subject) { - for (ListFileItem subItem : dir.listFilesAndDirectories()) { - delete(subItem); - } + } - dir.deleteIfExists(); - } else if (item.getClass() == CloudFile.class) { - CloudFile file = (CloudFile) item; + @Override + public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) { - file.deleteIfExists(); - } } @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { + public void remove(String noteId, String notePath, AuthenticationInfo subject) throws IOException { try { - CloudFileDirectory dir = rootDir.getDirectoryReference(noteId); - - delete(dir); + CloudFile noteFile = rootDir.getFileReference(buildNoteFileName(noteId, notePath)); + noteFile.delete(); } catch (URISyntaxException | StorageException e) { - String msg = String.format("Error deleting notebook %s from Azure storage", noteId); - - LOG.error(msg, e); - + String msg = String.format("Error deleting notebook %s from Azure storage", + buildNoteFileName(noteId, notePath)); + LOGGER.error(msg, e); throw new IOException(msg, e); } } @Override + public void remove(String folderPath, AuthenticationInfo subject) { + + } + + @Override public void close() { } @Override public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { - LOG.warn("Method not implemented"); + LOGGER.warn("Method not implemented"); return Collections.emptyList(); } @Override public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { - LOG.warn("Method not implemented"); + LOGGER.warn("Method not implemented"); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java index 5d9c85c..e7b03ca 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java @@ -1,12 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.notebook.repo; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.FileSystemStorage; import org.apache.zeppelin.notebook.Note; @@ -15,25 +26,14 @@ import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; /** * NotebookRepos for hdfs. * - * Assume the notebook directory structure is as following - * - notebookdir - * - noteId/note.json - * - noteId/note.json - * - noteId/note.json */ public class FileSystemNotebookRepo implements NotebookRepo { private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class); @@ -47,41 +47,73 @@ public class FileSystemNotebookRepo implements NotebookRepo { public void init(ZeppelinConfiguration zConf) throws IOException { this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir()); - LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() + - " for Zeppelin Notebook."); + LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName()); this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir())); LOGGER.info("Using folder {} to store notebook", notebookDir); this.fs.tryMkDir(notebookDir); } @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { - List<Path> notePaths = fs.list(new Path(notebookDir, "*/note.json")); - List<NoteInfo> noteInfos = new ArrayList<>(); + public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException { + List<Path> notePaths = fs.listAll(notebookDir); + Map<String, NoteInfo> noteInfos = new HashMap<>(); for (Path path : notePaths) { - NoteInfo noteInfo = new NoteInfo(path.getParent().getName(), "", null); - noteInfos.add(noteInfo); + try { + NoteInfo noteInfo = new NoteInfo(getNoteId(path.getName()), + getNotePath(notebookDir.toString(), path.toString())); + noteInfos.put(noteInfo.getId(), noteInfo); + } catch (IOException e) { + LOGGER.warn("Fail to get NoteInfo for note: " + path.getName(), e); + } } return noteInfos; } + @Override - public Note get(final String noteId, AuthenticationInfo subject) throws IOException { + public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException { String content = this.fs.readFile( - new Path(notebookDir.toString() + "/" + noteId + "/note.json")); + new Path(notebookDir, buildNoteFileName(noteId, notePath))); return Note.fromJson(content); } @Override - public void save(final Note note, AuthenticationInfo subject) throws IOException { + public void save(Note note, AuthenticationInfo subject) throws IOException { this.fs.writeFile(note.toJson(), - new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"), + new Path(notebookDir, buildNoteFileName(note.getId(), note.getPath())), true); } @Override - public void remove(final String noteId, AuthenticationInfo subject) throws IOException { - this.fs.delete(new Path(notebookDir.toString() + "/" + noteId)); + public void move(String noteId, + String notePath, + String newNotePath, + AuthenticationInfo subject) throws IOException { + Path src = new Path(notebookDir, buildNoteFileName(noteId, notePath)); + Path dest = new Path(notebookDir, buildNoteFileName(noteId, newNotePath)); + this.fs.move(src, dest); + } + + @Override + public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) + throws IOException { + this.fs.move(new Path(notebookDir, folderPath.substring(1)), + new Path(notebookDir, newFolderPath.substring(1))); + } + + @Override + public void remove(String noteId, String notePath, AuthenticationInfo subject) + throws IOException { + if (!this.fs.delete(new Path(notebookDir.toString(), buildNoteFileName(noteId, notePath)))) { + LOGGER.warn("Fail to move note, noteId: " + notePath + ", notePath: " + notePath); + } + } + + @Override + public void remove(String folderPath, AuthenticationInfo subject) throws IOException { + if (!this.fs.delete(new Path(notebookDir, folderPath.substring(1)))) { + LOGGER.warn("Fail to remove folder: " + folderPath); + } } @Override http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java index 5fd8becc..3301daa 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.zeppelin.notebook.repo; @@ -7,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.After; import org.junit.Before; @@ -52,7 +70,7 @@ public class FileSystemNotebookRepoTest { // create a new note Note note = new Note(); - note.setName("title_1"); + note.setPath("/title_1"); Map<String, Object> config = new HashMap<>(); config.put("config_1", "value_1"); @@ -61,20 +79,43 @@ public class FileSystemNotebookRepoTest { assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); // read this note from hdfs - Note note_copy = hdfsNotebookRepo.get(note.getId(), authInfo); + Note note_copy = hdfsNotebookRepo.get(note.getId(), note.getPath(), authInfo); assertEquals(note.getName(), note_copy.getName()); assertEquals(note.getConfig(), note_copy.getConfig()); // update this note - note.setName("title_2"); + note.setPersonalizedMode(true); hdfsNotebookRepo.save(note, authInfo); assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); - note_copy = hdfsNotebookRepo.get(note.getId(), authInfo); + note_copy = hdfsNotebookRepo.get(note.getId(), note.getPath(), authInfo); assertEquals(note.getName(), note_copy.getName()); assertEquals(note.getConfig(), note_copy.getConfig()); + // move this note + String newPath = "/new_folder/title_1"; + hdfsNotebookRepo.move(note.getId(), note.getPath(), newPath, authInfo); + assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); + assertEquals("title_1", hdfsNotebookRepo.get(note.getId(), newPath, authInfo).getName()); + // delete this note - hdfsNotebookRepo.remove(note.getId(), authInfo); + hdfsNotebookRepo.remove(note.getId(), newPath, authInfo); + assertEquals(0, hdfsNotebookRepo.list(authInfo).size()); + + // create another new note under folder + note = new Note(); + note.setPath("/folder1/title_1"); + note.setConfig(config); + hdfsNotebookRepo.save(note, authInfo); + assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); + + hdfsNotebookRepo.move("/folder1", "/folder2/folder3", authInfo); + Map<String, NoteInfo> notesInfo = hdfsNotebookRepo.list(authInfo); + assertEquals(1, notesInfo.size()); + + assertEquals("/folder2/folder3/title_1", notesInfo.get(note.getId()).getPath()); + + // delete folder + hdfsNotebookRepo.remove("/folder2", authInfo); assertEquals(0, hdfsNotebookRepo.list(authInfo).size()); } @@ -90,12 +131,11 @@ public class FileSystemNotebookRepoTest { // scenario_2: note_folder is existed. // create a new note Note note = new Note(); - note.setName("title_1"); + note.setPath("/title_1"); Map<String, Object> config = new HashMap<>(); config.put("config_1", "value_1"); note.setConfig(config); - fs.mkdirs(new Path(notebookDir, note.getId())); hdfsNotebookRepo.save(note, authInfo); assertEquals(1, hdfsNotebookRepo.list(authInfo).size()); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/filesystem/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/test/resources/log4j.properties b/zeppelin-plugins/notebookrepo/filesystem/src/test/resources/log4j.properties new file mode 100644 index 0000000..8daee59 --- /dev/null +++ b/zeppelin-plugins/notebookrepo/filesystem/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger = INFO, stdout + +log4j.appender.stdout = org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout = org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java index ad99aba..ee269df 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -30,9 +30,9 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.gson.JsonParseException; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -60,11 +60,11 @@ import org.slf4j.LoggerFactory; */ public class GCSNotebookRepo implements NotebookRepo { - private static final Logger LOG = LoggerFactory.getLogger(GCSNotebookRepo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(GCSNotebookRepo.class); private String encoding; private String bucketName; private Optional<String> basePath; - private Pattern noteNamePattern; + private Pattern notePathPattern; private Storage storage; public GCSNotebookRepo() { @@ -107,27 +107,27 @@ public class GCSNotebookRepo implements NotebookRepo { // Notes are stored at gs://bucketName/basePath/<note-id>/note.json if (basePath.isPresent()) { - this.noteNamePattern = Pattern.compile( - "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$"); + this.notePathPattern = Pattern.compile( + "^" + Pattern.quote(basePath.get() + "/") + "(.+\\.zpln)$"); } else { - this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$"); + this.notePathPattern = Pattern.compile("^(.+\\.zpln)$"); } this.storage = StorageOptions.getDefaultInstance().getService(); } - private BlobId makeBlobId(String noteId) { + private BlobId makeBlobId(String noteId, String notePath) throws IOException { if (basePath.isPresent()) { - return BlobId.of(bucketName, basePath.get() + "/" + noteId + "/note.json"); + return BlobId.of(bucketName, basePath.get() + "/" + buildNoteFileName(noteId, notePath)); } else { - return BlobId.of(bucketName, noteId + "/note.json"); + return BlobId.of(bucketName, buildNoteFileName(noteId, notePath)); } } @Override - public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { + public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException { try { - List<NoteInfo> infos = new ArrayList<>(); + Map<String, NoteInfo> infos = new HashMap<>(); Iterable<Blob> blobsUnderDir; if (basePath.isPresent()) { blobsUnderDir = storage @@ -139,11 +139,18 @@ public class GCSNotebookRepo implements NotebookRepo { .iterateAll(); } for (Blob b : blobsUnderDir) { - Matcher matcher = noteNamePattern.matcher(b.getName()); + Matcher matcher = notePathPattern.matcher(b.getName()); if (matcher.matches()) { // Callers only use the id field, so do not fetch each note // This matches the implementation in FileSystemNoteRepo#list - infos.add(new NoteInfo(matcher.group(1), "", null)); + String noteFileName = matcher.group(1); + try { + String noteId = getNoteId(noteFileName); + String notePath = getNotePath("", noteFileName); + infos.put(noteId, new NoteInfo(noteId, notePath)); + } catch (IOException e) { + LOGGER.warn(e.getMessage()); + } } } return infos; @@ -153,8 +160,8 @@ public class GCSNotebookRepo implements NotebookRepo { } @Override - public Note get(String noteId, AuthenticationInfo subject) throws IOException { - BlobId blobId = makeBlobId(noteId); + public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException { + BlobId blobId = makeBlobId(noteId, notePath); byte[] contents; try { contents = storage.readAllBytes(blobId); @@ -172,7 +179,7 @@ public class GCSNotebookRepo implements NotebookRepo { @Override public void save(Note note, AuthenticationInfo subject) throws IOException { - BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId(), note.getPath())) .setContentType("application/json") .build(); try { @@ -183,9 +190,19 @@ public class GCSNotebookRepo implements NotebookRepo { } @Override - public void remove(String noteId, AuthenticationInfo subject) throws IOException { + public void move(String noteId, String notePath, String newNotePath, AuthenticationInfo subject) { + + } + + @Override + public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) { + + } + + @Override + public void remove(String noteId, String notePath, AuthenticationInfo subject) throws IOException { Preconditions.checkArgument(!Strings.isNullOrEmpty(noteId)); - BlobId blobId = makeBlobId(noteId); + BlobId blobId = makeBlobId(noteId, notePath); try { boolean deleted = storage.delete(blobId); if (!deleted) { @@ -197,18 +214,23 @@ public class GCSNotebookRepo implements NotebookRepo { } @Override + public void remove(String folderPath, AuthenticationInfo subject) { + + } + + @Override public void close() { //no-op } @Override public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) { - LOG.warn("getSettings is not implemented for GCSNotebookRepo"); + LOGGER.warn("getSettings is not implemented for GCSNotebookRepo"); return Collections.emptyList(); } @Override public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) { - LOG.warn("updateSettings is not implemented for GCSNotebookRepo"); + LOGGER.warn("updateSettings is not implemented for GCSNotebookRepo"); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java index c1fae67..f2e19d0 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java @@ -32,6 +32,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; + import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; @@ -86,9 +88,10 @@ public class GCSNotebookRepoTest { private static Note makeRunningNote() { Note note = new Note(); + note.setPath("/test_note"); note.setConfig(ImmutableMap.<String, Object>of("key", "value")); - Paragraph p = new Paragraph(note, null, null); + Paragraph p = new Paragraph(note, null); p.setText("text"); p.setStatus(Status.RUNNING); note.addParagraph(p); @@ -102,26 +105,26 @@ public class GCSNotebookRepoTest { @Test public void testList() throws Exception { - createAt(runningNote, "note.json"); - createAt(runningNote, "/note.json"); - createAt(runningNote, "validid/note.json"); - createAt(runningNote, "validid-2/note.json"); + createAt(runningNote, "note.zpln"); + createAt(runningNote, "/note.zpln"); + createAt(runningNote, "validid/my_12.zpln"); + createAt(runningNote, "validid-2/my_123.zpln"); createAt(runningNote, "cannot-be-dir/note.json/foo"); createAt(runningNote, "cannot/be/nested/note.json"); - List<NoteInfo> infos = notebookRepo.list(AUTH_INFO); + Map<String, NoteInfo> infos = notebookRepo.list(AUTH_INFO); List<String> noteIds = new ArrayList<>(); - for (NoteInfo info : infos) { + for (NoteInfo info : infos.values()) { noteIds.add(info.getId()); } // Only valid paths are gs://bucketname/path/<noteid>/note.json - assertThat(noteIds).containsExactlyElementsIn(ImmutableList.of("validid", "validid-2")); + assertThat(noteIds).containsExactlyElementsIn(ImmutableList.of("12", "123")); } @Test public void testGet_nonexistent() throws Exception { try { - notebookRepo.get("id", AUTH_INFO); + notebookRepo.get("id", "", AUTH_INFO); fail(); } catch (IOException e) {} } @@ -131,7 +134,7 @@ public class GCSNotebookRepoTest { create(runningNote); // Status of saved running note is removed in get() - Note got = notebookRepo.get(runningNote.getId(), AUTH_INFO); + Note got = notebookRepo.get(runningNote.getId(), runningNote.getPath(), AUTH_INFO); assertThat(got.getLastParagraph().getStatus()).isEqualTo(Status.ABORT); // But otherwise equal @@ -141,9 +144,9 @@ public class GCSNotebookRepoTest { @Test public void testGet_malformed() throws Exception { - createMalformed("id"); + createMalformed("id", "/name"); try { - notebookRepo.get("id", AUTH_INFO); + notebookRepo.get("id", "/name", AUTH_INFO); fail(); } catch (IOException e) {} } @@ -152,7 +155,7 @@ public class GCSNotebookRepoTest { public void testSave_create() throws Exception { notebookRepo.save(runningNote, AUTH_INFO); // Output is saved - assertThat(storage.readAllBytes(makeBlobId(runningNote.getId()))) + assertThat(storage.readAllBytes(makeBlobId(runningNote.getId(), runningNote.getPath()))) .isEqualTo(runningNote.toJson().getBytes("UTF-8")); } @@ -160,16 +163,16 @@ public class GCSNotebookRepoTest { public void testSave_update() throws Exception { notebookRepo.save(runningNote, AUTH_INFO); // Change name of runningNote - runningNote.setName("new-name"); + runningNote.setPath("/new-name"); notebookRepo.save(runningNote, AUTH_INFO); - assertThat(storage.readAllBytes(makeBlobId(runningNote.getId()))) + assertThat(storage.readAllBytes(makeBlobId(runningNote.getId(), runningNote.getPath()))) .isEqualTo(runningNote.toJson().getBytes("UTF-8")); } @Test public void testRemove_nonexistent() throws Exception { try { - notebookRepo.remove("id", AUTH_INFO); + notebookRepo.remove("id", "/name", AUTH_INFO); fail(); } catch (IOException e) {} } @@ -177,8 +180,8 @@ public class GCSNotebookRepoTest { @Test public void testRemove() throws Exception { create(runningNote); - notebookRepo.remove(runningNote.getId(), AUTH_INFO); - assertThat(storage.get(makeBlobId(runningNote.getId()))).isNull(); + notebookRepo.remove(runningNote.getId(), runningNote.getPath(), AUTH_INFO); + assertThat(storage.get(makeBlobId(runningNote.getId(), runningNote.getPath()))).isNull(); } private String makeName(String relativePath) { @@ -189,8 +192,12 @@ public class GCSNotebookRepoTest { } } - private BlobId makeBlobId(String noteId) { - return BlobId.of(bucketName, makeName(noteId + "/note.json")); + private BlobId makeBlobId(String noteId, String notePath) { + if (basePath.isPresent()) { + return BlobId.of(bucketName, basePath.get() + notePath + "_" + noteId +".zpln"); + } else { + return BlobId.of(bucketName, notePath.substring(1) + "_" + noteId +".zpln"); + } } private void createAt(Note note, String relativePath) throws IOException { @@ -200,14 +207,14 @@ public class GCSNotebookRepoTest { } private void create(Note note) throws IOException { - BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId())) + BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId(), note.getPath())) .setContentType("application/json") .build(); storage.create(info, note.toJson().getBytes("UTF-8")); } - private void createMalformed(String noteId) throws IOException { - BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId)) + private void createMalformed(String noteId, String notePath) throws IOException { + BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId, notePath)) .setContentType("application/json") .build(); storage.create(info, "{ invalid-json }".getBytes("UTF-8")); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java index 7729d52..2da91fc 100644 --- a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java @@ -53,9 +53,8 @@ import java.util.List; * TODO(bzz): add default .gitignore */ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl { - private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(GitNotebookRepo.class); - private String localPath; private Git git; public GitNotebookRepo() { @@ -75,43 +74,74 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith this.conf = conf; setNotebookDirectory(conf.getNotebookDir()); - localPath = getRootDir().getName().getPath(); - LOG.info("Opening a git repo at '{}'", localPath); - Repository localRepo = new FileRepository(Joiner.on(File.separator).join(localPath, ".git")); + LOGGER.info("Opening a git repo at '{}'", this.rootNotebookFolder); + Repository localRepo = new FileRepository(Joiner.on(File.separator) + .join(this.rootNotebookFolder, ".git")); if (!localRepo.getDirectory().exists()) { - LOG.info("Git repo {} does not exist, creating a new one", localRepo.getDirectory()); + LOGGER.info("Git repo {} does not exist, creating a new one", localRepo.getDirectory()); localRepo.create(); } git = new Git(localRepo); } @Override - public synchronized void save(Note note, AuthenticationInfo subject) throws IOException { - super.save(note, subject); + public void move(String noteId, + String notePath, + String newNotePath, + AuthenticationInfo subject) throws IOException { + super.move(noteId, notePath, newNotePath, subject); + String noteFileName = buildNoteFileName(noteId, notePath); + String newNoteFileName = buildNoteFileName(noteId, newNotePath); + git.rm().addFilepattern(noteFileName); + git.add().addFilepattern(newNoteFileName); + try { + git.commit().setMessage("Move note " + noteId + " from " + noteFileName + " to " + + newNoteFileName).call(); + } catch (GitAPIException e) { + throw new IOException(e); + } + } + + @Override + public void move(String folderPath, String newFolderPath, + AuthenticationInfo subject) throws IOException { + super.move(folderPath, newFolderPath, subject); + git.rm().addFilepattern(folderPath.substring(1)); + git.add().addFilepattern(newFolderPath.substring(1)); + try { + git.commit().setMessage("Move folder " + folderPath + " to " + newFolderPath).call(); + } catch (GitAPIException e) { + throw new IOException(e); + } } /* implemented as git add+commit - * @param pattern is the noteId + * @param noteId is the noteId + * @param noteName name of the note * @param commitMessage is a commit message (checkpoint message) * (non-Javadoc) * @see org.apache.zeppelin.notebook.repo.VFSNotebookRepo#checkpoint(String, String) */ @Override - public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) { + public Revision checkpoint(String noteId, + String notePath, + String commitMessage, + AuthenticationInfo subject) throws IOException { + String noteFileName = buildNoteFileName(noteId, notePath); Revision revision = Revision.EMPTY; try { List<DiffEntry> gitDiff = git.diff().call(); if (!gitDiff.isEmpty()) { - LOG.debug("Changes found for pattern '{}': {}", pattern, gitDiff); - DirCache added = git.add().addFilepattern(pattern).call(); - LOG.debug("{} changes are about to be commited", added.getEntryCount()); + LOGGER.debug("Changes found for pattern '{}': {}", noteFileName, gitDiff); + DirCache added = git.add().addFilepattern(noteFileName).call(); + LOGGER.debug("{} changes are about to be commited", added.getEntryCount()); RevCommit commit = git.commit().setMessage(commitMessage).call(); revision = new Revision(commit.getName(), commit.getShortMessage(), commit.getCommitTime()); } else { - LOG.debug("No changes found {}", pattern); + LOGGER.debug("No changes found {}", noteFileName); } } catch (GitAPIException e) { - LOG.error("Failed to add+commit {} to Git", pattern, e); + LOGGER.error("Failed to add+commit {} to Git", noteFileName, e); } return revision; } @@ -124,63 +154,70 @@ public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWith * 4. apply stash on top and remove it */ @Override - public synchronized Note get(String noteId, String revId, AuthenticationInfo subject) - throws IOException { + public synchronized Note get(String noteId, + String notePath, + String revId, + AuthenticationInfo subject) throws IOException { Note note = null; RevCommit stash = null; + String noteFileName = buildNoteFileName(noteId, notePath); try { - List<DiffEntry> gitDiff = git.diff().setPathFilter(PathFilter.create(noteId)).call(); + List<DiffEntry> gitDiff = git.diff().setPathFilter(PathFilter.create(noteFileName)).call(); boolean modified = !gitDiff.isEmpty(); if (modified) { // stash changes stash = git.stashCreate().call(); Collection<RevCommit> stashes = git.stashList().call(); - LOG.debug("Created stash : {}, stash size : {}", stash, stashes.size()); + LOGGER.debug("Created stash : {}, stash size : {}", stash, stashes.size()); } ObjectId head = git.getRepository().resolve(Constants.HEAD); // checkout to target revision - git.checkout().setStartPoint(revId).addPath(noteId).call(); + git.checkout().setStartPoint(revId).addPath(noteFileName).call(); // get the note - note = super.get(noteId, subject); + note = super.get(noteId, notePath, subject); // checkout back to head - git.checkout().setStartPoint(head.getName()).addPath(noteId).call(); + git.checkout().setStartPoint(head.getName()).addPath(noteFileName).call(); if (modified && stash != null) { // unstash changes ObjectId applied = git.stashApply().setStashRef(stash.getName()).call(); ObjectId dropped = git.stashDrop().setStashRef(0).call(); Collection<RevCommit> stashes = git.stashList().call(); - LOG.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped, + LOGGER.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped, stashes.size()); } } catch (GitAPIException e) { - LOG.error("Failed to return note from revision \"{}\"", revId, e); + LOGGER.error("Failed to return note from revision \"{}\"", revId, e); } return note; } @Override - public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) { + public List<Revision> revisionHistory(String noteId, + String notePath, + AuthenticationInfo subject) throws IOException { List<Revision> history = Lists.newArrayList(); - LOG.debug("Listing history for {}:", noteId); + String noteFileName = buildNoteFileName(noteId, notePath); + LOGGER.debug("Listing history for {}:", noteFileName); try { - Iterable<RevCommit> logs = git.log().addPath(noteId).call(); + Iterable<RevCommit> logs = git.log().addPath(noteFileName).call(); for (RevCommit log: logs) { history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime())); - LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage()); + LOGGER.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage()); } } catch (NoHeadException e) { //when no initial commit exists - LOG.warn("No Head found for {}, {}", noteId, e.getMessage()); + LOGGER.warn("No Head found for {}, {}", noteFileName, e.getMessage()); } catch (GitAPIException e) { - LOG.error("Failed to get logs for {}", noteId, e); + LOGGER.error("Failed to get logs for {}", noteFileName, e); } return history; } @Override - public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject) + public Note setNoteRevision(String noteId, String noteName, String revId, + AuthenticationInfo subject) throws IOException { - Note revisionNote = get(noteId, revId, subject); + Note revisionNote = get(noteId, noteName, revId, subject); if (revisionNote != null) { save(revisionNote, subject); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/085efeb6/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java index 58d7cc1..6c30e42 100644 --- a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java @@ -51,6 +51,8 @@ public class GitNotebookRepoTest { private static final String TEST_NOTE_ID = "2A94M5J1Z"; private static final String TEST_NOTE_ID2 = "2A94M5J2Z"; + private static final String TEST_NOTE_PATH = "/my_project/my_note1"; + private static final String TEST_NOTE_PATH2 = "/my_project/my_note2"; private File zeppelinDir; private String notebooksDir; @@ -68,23 +70,9 @@ public class GitNotebookRepoTest { File notebookDir = new File(notebooksDir); notebookDir.mkdirs(); - String testNoteDir = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID); - String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2); FileUtils.copyDirectory( - new File( - GitNotebookRepoTest.class.getResource( - Joiner.on(File.separator).join("", TEST_NOTE_ID) - ).getFile() - ), - new File(testNoteDir)); - FileUtils.copyDirectory( - new File( - GitNotebookRepoTest.class.getResource( - Joiner.on(File.separator).join("", TEST_NOTE_ID2) - ).getFile() - ), - new File(testNoteDir2) - ); + new File(GitNotebookRepoTest.class.getResource("/notebook").getFile()), + new File(notebooksDir)); System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinDir.getAbsolutePath()); System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); @@ -128,7 +116,7 @@ public class GitNotebookRepoTest { assertThat(notebookRepo.list(null)).isNotEmpty(); //when - List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, null); + List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null); //then //no initial commit, empty history @@ -142,17 +130,17 @@ public class GitNotebookRepoTest { assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID2)).isTrue(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, TEST_NOTE_PATH2, null)).isEmpty(); //add commit to both notes - notebookRepo.checkpoint(TEST_NOTE_ID, "first commit, note1", null); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); - notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); + notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "first commit, note1", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(1); + notebookRepo.checkpoint(TEST_NOTE_ID2, TEST_NOTE_PATH2, "first commit, note2", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, TEST_NOTE_PATH2, null).size()).isEqualTo(1); //modify, save and checkpoint first note - Note note = notebookRepo.get(TEST_NOTE_ID, null); + Note note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map<String, Object> config = p.getConfig(); @@ -160,15 +148,15 @@ public class GitNotebookRepoTest { p.setConfig(config); p.setText("%md note1 test text"); notebookRepo.save(note, null); - assertThat(notebookRepo.checkpoint(TEST_NOTE_ID, "second commit, note1", null)).isNotNull(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); - assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null)) + assertThat(notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "second commit, note1", null)).isNotNull(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(2); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, TEST_NOTE_PATH2, null).size()).isEqualTo(1); + assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, TEST_NOTE_PATH2, "first commit, note2", null)) .isEqualTo(Revision.EMPTY); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, TEST_NOTE_PATH2, null).size()).isEqualTo(1); //modify, save and checkpoint second note - note = notebookRepo.get(TEST_NOTE_ID2, null); + note = notebookRepo.get(TEST_NOTE_ID2, TEST_NOTE_PATH2, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); config = p.getConfig(); @@ -176,9 +164,9 @@ public class GitNotebookRepoTest { p.setConfig(config); p.setText("%md note2 test text"); notebookRepo.save(note, null); - assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "second commit, note2", null)).isNotNull(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(2); + assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, TEST_NOTE_PATH2, "second commit, note2", null)).isNotNull(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(2); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, TEST_NOTE_PATH2, null).size()).isEqualTo(2); } @Test @@ -187,15 +175,15 @@ public class GitNotebookRepoTest { notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null)).isEmpty(); - notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null); - List<Revision> notebookHistoryBefore = notebookRepo.revisionHistory(TEST_NOTE_ID, null); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isNotEmpty(); + notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "first commit", null); + List<Revision> notebookHistoryBefore = notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null)).isNotEmpty(); int initialCount = notebookHistoryBefore.size(); // add changes to note - Note note = notebookRepo.get(TEST_NOTE_ID, null); + Note note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map<String, Object> config = p.getConfig(); @@ -205,15 +193,15 @@ public class GitNotebookRepoTest { // save and checkpoint note notebookRepo.save(note, null); - notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null); + notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "second commit", null); // see if commit is added - List<Revision> notebookHistoryAfter = notebookRepo.revisionHistory(TEST_NOTE_ID, null); + List<Revision> notebookHistoryAfter = notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null); assertThat(notebookHistoryAfter.size()).isEqualTo(initialCount + 1); } - private boolean containsNote(List<NoteInfo> notes, String noteId) { - for (NoteInfo note: notes) { + private boolean containsNote(Map<String, NoteInfo> notes, String noteId) { + for (NoteInfo note: notes.values()) { if (note.getId().equals(noteId)) { return true; } @@ -227,15 +215,15 @@ public class GitNotebookRepoTest { notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null)).isEmpty(); // add first checkpoint - Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); - int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size(); + Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "first commit", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(1); + int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null).getParagraphs().size(); // add paragraph and save - Note note = notebookRepo.get(TEST_NOTE_ID, null); + Note note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); Map<String, Object> config = p1.getConfig(); @@ -245,17 +233,17 @@ public class GitNotebookRepoTest { notebookRepo.save(note, null); // second checkpoint - notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); - int paragraphCount_2 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size(); + notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "second commit", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(2); + int paragraphCount_2 = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null).getParagraphs().size(); assertThat(paragraphCount_2).isEqualTo(paragraphCount_1 + 1); // get note from revision 1 - Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null); + Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, revision_1.id, null); assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1); // get current note - note = notebookRepo.get(TEST_NOTE_ID, null); + note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2); @@ -265,17 +253,17 @@ public class GitNotebookRepoTest { p2.setConfig(config); p2.setText("get revision when modified note test text"); notebookRepo.save(note, null); - note = notebookRepo.get(TEST_NOTE_ID, null); + note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); int paragraphCount_3 = note.getParagraphs().size(); assertThat(paragraphCount_3).isEqualTo(paragraphCount_2 + 1); // get revision 1 again - noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null); + noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, revision_1.id, null); assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1); // check that note is unchanged - note = notebookRepo.get(TEST_NOTE_ID, null); + note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_3); } @@ -285,15 +273,15 @@ public class GitNotebookRepoTest { notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null)).isEmpty(); // add first checkpoint - Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); - int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size(); + Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "first commit", null); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(1); + int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null).getParagraphs().size(); // get current note - Note note = notebookRepo.get(TEST_NOTE_ID, null); + Note note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_1); @@ -307,17 +295,17 @@ public class GitNotebookRepoTest { int paragraphCount_2 = note.getParagraphs().size(); // get note from revision 1 - Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null); + Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, revision_1.id, null); assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1); // get current note - note = notebookRepo.get(TEST_NOTE_ID, null); + note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2); // test for absent revision Revision absentRevision = new Revision("absentId", StringUtils.EMPTY, 0); - note = notebookRepo.get(TEST_NOTE_ID, absentRevision.id, null); + note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, absentRevision.id, null); assertThat(note).isNull(); } @@ -327,19 +315,19 @@ public class GitNotebookRepoTest { notebookRepo = new GitNotebookRepo(conf); assertThat(notebookRepo.list(null)).isNotEmpty(); assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty(); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null)).isEmpty(); // get current note - Note note = notebookRepo.get(TEST_NOTE_ID, null); + Note note = notebookRepo.get(TEST_NOTE_ID, TEST_NOTE_PATH, null); note.setInterpreterFactory(mock(InterpreterFactory.class)); int paragraphCount_1 = note.getParagraphs().size(); LOG.info("initial paragraph count: {}", paragraphCount_1); // checkpoint revision1 - Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: first commit", null); + Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "set revision: first commit", null); //TODO(khalid): change to EMPTY after rebase assertThat(revision1).isNotNull(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(1); // add one more paragraph and save Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -353,33 +341,33 @@ public class GitNotebookRepoTest { LOG.info("paragraph count after modification: {}", paragraphCount_2); // checkpoint revision2 - Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: second commit", null); + Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, TEST_NOTE_PATH, "set revision: second commit", null); //TODO(khalid): change to EMPTY after rebase assertThat(revision2).isNotNull(); - assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2); + assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, TEST_NOTE_PATH, null).size()).isEqualTo(2); // set note to revision1 - Note returnedNote = notebookRepo.setNoteRevision(note.getId(), revision1.id, null); + Note returnedNote = notebookRepo.setNoteRevision(note.getId(), note.getPath(), revision1.id, null); assertThat(returnedNote).isNotNull(); assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_1); // check note from repo - Note updatedNote = notebookRepo.get(note.getId(), null); + Note updatedNote = notebookRepo.get(note.getId(), note.getPath(), null); assertThat(updatedNote).isNotNull(); assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_1); // set back to revision2 - returnedNote = notebookRepo.setNoteRevision(note.getId(), revision2.id, null); + returnedNote = notebookRepo.setNoteRevision(note.getId(), note.getPath(), revision2.id, null); assertThat(returnedNote).isNotNull(); assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_2); // check note from repo - updatedNote = notebookRepo.get(note.getId(), null); + updatedNote = notebookRepo.get(note.getId(), note.getPath(), null); assertThat(updatedNote).isNotNull(); assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_2); // try failure case - set to invalid revision - returnedNote = notebookRepo.setNoteRevision(note.getId(), "nonexistent_id", null); + returnedNote = notebookRepo.setNoteRevision(note.getId(), note.getPath(), "nonexistent_id", null); assertThat(returnedNote).isNull(); } }
