Repository: zeppelin Updated Branches: refs/heads/branch-0.7 54eb18f41 -> abdf3352e
[HOTFIX] Forward compatibility of Note class ### What is this PR for? Solving inconsistency issue of Note's dateformat between branch-0.7 and master. It's related to #2272 ### What type of PR is it? [Hot Fix] ### Todos * [x] - Support several dateformat for Note class ### What is the Jira issue? * N/A ### How should this be tested? * Sync with some remote repository ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jongyoul Lee <jongy...@gmail.com> Closes #2591 from jongyoul/hotfix/note-json-forward-compatibility and squashes the following commits: 046a245e [Jongyoul Lee] Fixed style 492e7d30 [Jongyoul Lee] Make test case more concrete 13b9aef5 [Jongyoul Lee] Added one more deserialize option 53add839 [Jongyoul Lee] Hotfix for forward compatibility of Note.class Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/abdf3352 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/abdf3352 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/abdf3352 Branch: refs/heads/branch-0.7 Commit: abdf3352e5817e2da0de29f51b72d761cabb4919 Parents: 54eb18f Author: Jongyoul Lee <jongy...@gmail.com> Authored: Mon Sep 18 13:53:44 2017 +0900 Committer: Jongyoul Lee <jongy...@apache.org> Committed: Mon Sep 18 17:01:19 2017 +0900 ---------------------------------------------------------------------- .../java/org/apache/zeppelin/notebook/Note.java | 42 +++++++++++----- .../org/apache/zeppelin/notebook/Notebook.java | 2 +- .../notebook/NotebookImportDeserializer.java | 12 ++--- .../notebook/repo/AzureNotebookRepo.java | 6 +-- .../zeppelin/notebook/repo/S3NotebookRepo.java | 2 +- .../zeppelin/notebook/repo/VFSNotebookRepo.java | 2 +- .../repo/zeppelinhub/ZeppelinHubRepo.java | 4 +- .../apache/zeppelin/notebook/ParagraphTest.java | 50 +++++++++++--------- 8 files changed, 69 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 1964a08..ae0de5d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -17,21 +17,37 @@ package org.apache.zeppelin.notebook; -import static java.lang.String.format; - +import com.google.common.base.Preconditions; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; - import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.notebook.repo.NotebookRepo; @@ -42,11 +58,8 @@ import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; -import com.google.gson.Gson; +import static java.lang.String.format; /** * Binded interpreters for a note @@ -59,6 +72,10 @@ public class Note implements Serializable, ParagraphJobListener { private static final ScheduledThreadPoolExecutor delayedPersistThreadPool = new ScheduledThreadPoolExecutor(0); + public static final Gson GSON = new GsonBuilder().setPrettyPrinting() + .setDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); + static { delayedPersistThreadPool.setRemoveOnCancelPolicy(true); } @@ -323,9 +340,8 @@ public class Note implements Serializable, ParagraphJobListener { newParagraph.setTitle(srcParagraph.getTitle()); try { - Gson gson = new Gson(); - String resultJson = gson.toJson(srcParagraph.getReturn()); - InterpreterResult result = gson.fromJson(resultJson, InterpreterResult.class); + String resultJson = GSON.toJson(srcParagraph.getReturn()); + InterpreterResult result = GSON.fromJson(resultJson, InterpreterResult.class); newParagraph.setReturn(result, null); } catch (Exception e) { // 'result' part of Note consists of exception, instead of actual interpreter results @@ -589,7 +605,7 @@ public class Note implements Serializable, ParagraphJobListener { public void run(String paragraphId) { Paragraph p = getParagraph(paragraphId); p.setListener(jobListenerFactory.getParagraphJobListener(this)); - + if (p.isBlankParagraph()) { logger.info("skip to run blank paragraph. {}", p.getId()); p.setStatus(Job.Status.FINISHED); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 953a73f..e8739f4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -211,7 +211,7 @@ public class Notebook implements NoteEventListener { reader.setLenient(true); Note newNote; try { - Note oldNote = gson.fromJson(reader, Note.class); + Note oldNote = Note.GSON.fromJson(reader, Note.class); convertFromSingleResultToMultipleResultsFormat(oldNote); newNote = createNote(subject); if (noteName != null) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java index 86e0f0b..7211039 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java @@ -21,7 +21,6 @@ import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonParseException; - import java.lang.reflect.Type; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -30,18 +29,19 @@ import java.util.Date; import java.util.Locale; /** - * importNote date format deserializer + * importNote date format deserializer */ public class NotebookImportDeserializer implements JsonDeserializer<Date> { - private static final String[] DATE_FORMATS = new String[] { + private static final String[] DATE_FORMATS = new String[]{ "yyyy-MM-dd'T'HH:mm:ssZ", "MMM d, yyyy h:mm:ss a", - "MMM dd, yyyy HH:mm:ss" + "MMM dd, yyyy HH:mm:ss", + "yyyy-MM-dd HH:mm:ss.SSS" }; @Override public Date deserialize(JsonElement jsonElement, Type typeOF, - JsonDeserializationContext context) throws JsonParseException { + JsonDeserializationContext context) throws JsonParseException { for (String format : DATE_FORMATS) { try { return new SimpleDateFormat(format, Locale.US).parse(jsonElement.getAsString()); @@ -49,6 +49,6 @@ public class NotebookImportDeserializer implements JsonDeserializer<Date> { } } throw new JsonParseException("Unparsable date: \"" + jsonElement.getAsString() - + "\". Supported formats: " + Arrays.toString(DATE_FORMATS)); + + "\". Supported formats: " + Arrays.toString(DATE_FORMATS)); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java index 79f5dd6..1ccc78f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java @@ -25,7 +25,6 @@ import java.io.Writer; import java.net.URISyntaxException; import java.security.InvalidKeyException; import java.util.Collections; -import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -35,7 +34,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; -import org.apache.zeppelin.notebook.NotebookImportDeserializer; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.user.AuthenticationInfo; @@ -135,10 +133,8 @@ public class AzureNotebookRepo implements NotebookRepo { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); - Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()) - .create(); - Note note = gson.fromJson(json, Note.class); + Note note = Note.GSON.fromJson(json, Note.class); for (Paragraph p : note.getParagraphs()) { if (p.getStatus() == Job.Status.PENDING || p.getStatus() == Job.Status.RUNNING) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 26781b8..314c14e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -199,7 +199,7 @@ public class S3NotebookRepo implements NotebookRepo { Note note; try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); - note = gson.fromJson(json, Note.class); + note = Note.GSON.fromJson(json, Note.class); } for (Paragraph p : note.getParagraphs()) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 04a7075..aebda97 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -175,7 +175,7 @@ public class VFSNotebookRepo implements NotebookRepo { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); ins.close(); - Note note = gson.fromJson(json, Note.class); + Note note = Note.GSON.fromJson(json, Note.class); // note.setReplLoader(replLoader); // note.jobListenerFactory = jobListenerFactory; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java index cd94180..a4e41a8 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java @@ -186,7 +186,7 @@ public class ZeppelinHubRepo implements NotebookRepo { } String token = getUserToken(subject.getUser()); String response = restApiClient.get(token, noteId); - Note note = GSON.fromJson(response, Note.class); + Note note = Note.GSON.fromJson(response, Note.class); if (note == null) { return EMPTY_NOTE; } @@ -245,7 +245,7 @@ public class ZeppelinHubRepo implements NotebookRepo { String token = getUserToken(subject.getUser()); String response = restApiClient.get(token, endpoint); - Note note = GSON.fromJson(response, Note.class); + Note note = Note.GSON.fromJson(response, Note.class); if (note == null) { return EMPTY_NOTE; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index 0e77846..c7d2a6a 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -18,22 +18,16 @@ package org.apache.zeppelin.notebook; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import com.google.common.collect.Lists; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.junit.Test; +import org.mockito.Mockito; + import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectBuilder; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -41,7 +35,6 @@ import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.Interpreter.FormType; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterResult; @@ -52,15 +45,19 @@ import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.InterpreterSetting.Status; import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.resource.ResourcePool; -import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; -import org.junit.Test; -import java.util.HashMap; -import java.util.Map; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class ParagraphTest { @Test @@ -84,7 +81,7 @@ public class ParagraphTest { assertEquals("md", Paragraph.getRequiredReplName(text)); assertEquals("", Paragraph.getScriptBody(text)); } - + @Test public void replSingleCharName() { String text = "%r a"; @@ -232,4 +229,13 @@ public class ParagraphTest { } + + @Test + public void deserializeParagraphFromDifferentDateFormat() throws ParseException { + String dateString = "2017-09-01 00:01:02.345"; + Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(dateString); + Paragraph paragraph = Note.GSON + .fromJson("{\"dateUpdated\":\"" + dateString + "\"}", Paragraph.class); + assertEquals(date, paragraph.dateUpdated); + } }
