Repository: incubator-zeppelin Updated Branches: refs/heads/master 81b47c039 -> a87d45ec0
[ZEPPELIN-599]notebook search should search paragraph title ### What is this PR for? Allow notebook search to search paragraph title too. ### What type of PR is it? [Bug Fix] ### Todos ### What is the Jira issue? [ZEPPELIN-599](https://issues.apache.org/jira/browse/ZEPPELIN-599?jql=project%20%3D%20ZEPPELIN%20AND%20status%20%3D%20Open%20AND%20text%20~%20%22Title%22) ### How should this be tested? You should be able to search the the note by searching queryterm in the paragraph title. ### Screenshots (if appropriate) Before:  After:  ### Questions: * Does the licenses files need update?NO * Is there breaking changes for older versions?NO * Does this needs documentation?NO Author: Ravi Ranjan <[email protected]> Closes #859 from ravicodder/searchTitle and squashes the following commits: 59b5a76 [Ravi Ranjan] Make code consistent c8a9eaf [Ravi Ranjan] Merge branch 'master' of https://github.com/apache/incubator-zeppelin into searchTitle cbded8d [Ravi Ranjan] Add test in LuceneSearchTest.java b3e66bb [Ravi Ranjan] Add test f6d64fd [Ravi Ranjan] Fix spacing 5830c8f [Ravi Ranjan] revert indent ed69177 [Ravi Ranjan] Search Title of notebook Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/a87d45ec Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/a87d45ec Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/a87d45ec Branch: refs/heads/master Commit: a87d45ec0460c64d709f8bc67e847bf17dc8f9d3 Parents: 81b47c0 Author: Ravi Ranjan <[email protected]> Authored: Thu May 5 10:53:15 2016 +0530 Committer: Prabhjyot Singh <[email protected]> Committed: Wed May 11 12:04:06 2016 +0530 ---------------------------------------------------------------------- .../zeppelin/rest/ZeppelinRestApiTest.java | 26 ++++++++ .../src/app/search/result-list.controller.js | 36 ++++++++-- zeppelin-web/src/app/search/search.css | 5 ++ .../apache/zeppelin/search/LuceneSearch.java | 52 ++++++++++----- .../zeppelin/search/LuceneSearchTest.java | 69 +++++++++++++++----- 5 files changed, 149 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 2f2a36b..36c95af 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -756,5 +756,31 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi { ZeppelinServer.notebook.removeNote(note2.getId()); } + @Test + public void testTitleSearch() throws IOException { + Note note = ZeppelinServer.notebook.createNote(); + String jsonRequest = "{\"title\": \"testTitleSearchOfParagraph\", \"text\": \"ThisIsToTestSearchMethodWithTitle \"}"; + PostMethod postNotebookText = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest); + postNotebookText.releaseConnection(); + + GetMethod searchNotebook = httpGet("/notebook/search?q='testTitleSearchOfParagraph'"); + searchNotebook.addRequestHeader("Origin", "http://localhost";); + Map<String, Object> respSearchResult = gson.fromJson(searchNotebook.getResponseBodyAsString(), + new TypeToken<Map<String, Object>>() { + }.getType()); + ArrayList searchBody = (ArrayList) respSearchResult.get("body"); + + int numberOfTitleHits = 0; + for (int i = 0; i < searchBody.size(); i++) { + Map<String, String> searchResult = (Map<String, String>) searchBody.get(i); + if (searchResult.get("header").contains("testTitleSearchOfParagraph")) { + numberOfTitleHits++; + } + } + assertEquals("Paragraph title hits must be at-least one", true, numberOfTitleHits >= 1); + searchNotebook.releaseConnection(); + ZeppelinServer.notebook.removeNote(note.getId()); + } + } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-web/src/app/search/result-list.controller.js ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/search/result-list.controller.js b/zeppelin-web/src/app/search/result-list.controller.js index 0d55442..949e01f 100644 --- a/zeppelin-web/src/app/search/result-list.controller.js +++ b/zeppelin-web/src/app/search/result-list.controller.js @@ -74,12 +74,20 @@ angular }; } - var lines = note.snippet + var result = ''; + if (note.header !== '') { + result = note.header + '\n\n' + note.snippet; + } else { + result = note.snippet; + } + + var lines = result .split('\n') .map(function(line, row) { + var match = line.match(/<B>(.+?)<\/B>/); - // return early if nothing to highlight + // return early if nothing to highlight if (!match) { return line; } @@ -93,15 +101,31 @@ angular indeces.forEach(function(start) { var end = start + term.length; - _editor - .getSession() - .addMarker( + if (note.header !== '' && row === 0) { + _editor + .getSession() + .addMarker( + new Range(row, 0, row, line.length), + 'search-results-highlight-header', + 'background' + ); + _editor + .getSession() + .addMarker( new Range(row, start, row, end), 'search-results-highlight', 'line' ); + } else { + _editor + .getSession() + .addMarker( + new Range(row, start, row, end), + 'search-results-highlight', + 'line' + ); + } }); - return __line; }); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-web/src/app/search/search.css ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/search/search.css b/zeppelin-web/src/app/search/search.css index e89c765..b06b4a9 100644 --- a/zeppelin-web/src/app/search/search.css +++ b/zeppelin-web/src/app/search/search.css @@ -31,6 +31,11 @@ position: absolute; } +.search-results-highlight-header { + background-color: #e6f2ff; + position: absolute; +} + /* remove error highlighting */ .search-results .ace_invalid { background: none !important; http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java index 7f9cbbd..b43c453 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; @@ -69,7 +70,8 @@ import com.google.common.collect.Lists; public class LuceneSearch implements SearchService { private static final Logger LOG = LoggerFactory.getLogger(LuceneSearch.class); - private static final String SEARCH_FIELD = "contents"; + private static final String SEARCH_FIELD_TEXT = "contents"; + private static final String SEARCH_FIELD_TITLE = "header"; static final String PARAGRAPH = "paragraph"; static final String ID_FIELD = "id"; @@ -85,7 +87,7 @@ public class LuceneSearch implements SearchService { try { writer = new IndexWriter(ramDirectory, iwc); } catch (IOException e) { - LOG.error("Failed to reate new IndexWriter", e); + LOG.error("Failed to create new IndexWriter", e); } } @@ -102,10 +104,12 @@ public class LuceneSearch implements SearchService { try (IndexReader indexReader = DirectoryReader.open(ramDirectory)) { IndexSearcher indexSearcher = new IndexSearcher(indexReader); Analyzer analyzer = new StandardAnalyzer(); - QueryParser parser = new QueryParser(SEARCH_FIELD, analyzer); + MultiFieldQueryParser parser = new MultiFieldQueryParser( + new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE}, + analyzer); Query query = parser.parse(queryStr); - LOG.debug("Searching for: " + query.toString(SEARCH_FIELD)); + LOG.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT)); SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter(); Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query)); @@ -139,20 +143,33 @@ public class LuceneSearch implements SearchService { LOG.debug(" Title: {}", doc.get("title")); } - String text = doc.get(SEARCH_FIELD); - TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id, - SEARCH_FIELD, analyzer); - TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3); - LOG.debug(" {} fragments found for query '{}'", frag.length, query); - for (int j = 0; j < frag.length; j++) { - if ((frag[j] != null) && (frag[j].getScore() > 0)) { - LOG.debug(" Fragment: {}", frag[j].toString()); + String text = doc.get(SEARCH_FIELD_TEXT); + String header = doc.get(SEARCH_FIELD_TITLE); + String fragment = ""; + + if (text != null) { + TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id, + SEARCH_FIELD_TEXT, analyzer); + TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3); + LOG.debug(" {} fragments found for query '{}'", frag.length, query); + for (int j = 0; j < frag.length; j++) { + if ((frag[j] != null) && (frag[j].getScore() > 0)) { + LOG.debug(" Fragment: {}", frag[j].toString()); + } } + fragment = (frag != null && frag.length > 0) ? frag[0].toString() : ""; } - String fragment = (frag != null && frag.length > 0) ? frag[0].toString() : ""; + if (header != null) { + TokenStream tokenTitle = TokenSources.getTokenStream(searcher.getIndexReader(), id, + SEARCH_FIELD_TITLE, analyzer); + TextFragment[] frgTitle = highlighter.getBestTextFragments(tokenTitle, header, true, 3); + header = (frgTitle != null && frgTitle.length > 0) ? frgTitle[0].toString() : ""; + } else { + header = ""; + } matchingParagraphs.add(ImmutableMap.of("id", path, // <noteId>/paragraph/<paragraphId> - "name", title, "snippet", fragment, "text", text)); + "name", title, "snippet", fragment, "text", text, "header", header)); } else { LOG.info("{}. No {} for this document", i + 1, ID_FIELD); } @@ -252,11 +269,14 @@ public class LuceneSearch implements SearchService { doc.add(new StringField("title", noteName, Field.Store.YES)); if (null != p) { - doc.add(new TextField(SEARCH_FIELD, p.getText(), Field.Store.YES)); + doc.add(new TextField(SEARCH_FIELD_TEXT, p.getText(), Field.Store.YES)); + if (p.getTitle() != null) { + doc.add(new TextField(SEARCH_FIELD_TITLE, p.getTitle(), Field.Store.YES)); + } Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated(); doc.add(new LongField("modified", date.getTime(), Field.Store.NO)); } else { - doc.add(new TextField(SEARCH_FIELD, noteName, Field.Store.YES)); + doc.add(new TextField(SEARCH_FIELD_TEXT, noteName, Field.Store.YES)); } return doc; } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/a87d45ec/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index f74d95e..c744267 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -65,8 +65,8 @@ public class LuceneSearchTest { @Test public void canIndexNotebook() { //give - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraph("Notebook2", "not test"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraph("Notebook2", "not test"); List<Note> notebook = Arrays.asList(note1, note2); //when @@ -75,8 +75,8 @@ public class LuceneSearchTest { @Test public void canIndexAndQuery() { //given - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); //when @@ -91,8 +91,8 @@ public class LuceneSearchTest { @Test public void canIndexAndQueryByNotebookName() { //given - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); //when @@ -104,9 +104,31 @@ public class LuceneSearchTest { assertThat(results.get(0)).containsEntry("id", note1.getId()); } + @Test + public void canIndexAndQueryByParagraphTitle() { + //given + Note note1 = newNoteWithParagraph("Notebook1", "test", "testingTitleSearch"); + Note note2 = newNoteWithParagraph("Notebook2", "not test", "notTestingTitleSearch"); + notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); + + //when + List<Map<String, String>> results = notebookIndex.query("testingTitleSearch"); + + //then + assertThat(results).isNotEmpty(); + assertThat(results.size()).isAtLeast(1); + int TitleHits = 0; + for (Map<String, String> res : results) { + if (res.get("header").contains("testingTitleSearch")) { + TitleHits++; + } + } + assertThat(TitleHits).isAtLeast(1); + } + @Test public void indexKeyContract() throws IOException { //give - Note note1 = newNoteWithParapgraph("Notebook1", "test"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); //when notebookIndex.addIndexDoc(note1); //then @@ -129,8 +151,8 @@ public class LuceneSearchTest { @Test public void canIndexAndReIndex() throws IOException { //given - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); //when @@ -155,8 +177,8 @@ public class LuceneSearchTest { @Test public void canDeleteFromIndex() throws IOException { //given - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); assertThat(resultForQuery("Notebook2")).isNotEmpty(); @@ -174,8 +196,8 @@ public class LuceneSearchTest { @Test public void indexParagraphUpdatedOnNoteSave() throws IOException { //given: total 2 notebooks, 3 paragraphs - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); assertThat(resultForQuery("test").size()).isEqualTo(3); @@ -199,8 +221,8 @@ public class LuceneSearchTest { @Test public void indexNoteNameUpdatedOnNoteSave() throws IOException { //given: total 2 notebooks, 3 paragraphs - Note note1 = newNoteWithParapgraph("Notebook1", "test"); - Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all"); + Note note1 = newNoteWithParagraph("Notebook1", "test"); + Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all"); notebookIndex.addIndexDocs(Arrays.asList(note1, note2)); assertThat(resultForQuery("test").size()).isEqualTo(3); @@ -226,17 +248,23 @@ public class LuceneSearchTest { * @param parText text of the paragraph * @return Note */ - private Note newNoteWithParapgraph(String noteName, String parText) { + private Note newNoteWithParagraph(String noteName, String parText) { Note note1 = newNote(noteName); addParagraphWithText(note1, parText); return note1; } + private Note newNoteWithParagraph(String noteName, String parText,String title) { + Note note = newNote(noteName); + addParagraphWithTextAndTitle(note, parText, title); + return note; + } + /** * Creates a new Note \w given name, * adds N paragraphs \w given texts */ - private Note newNoteWithParapgraphs(String noteName, String... parTexts) { + private Note newNoteWithParagraphs(String noteName, String... parTexts) { Note note1 = newNote(noteName); for (String parText : parTexts) { addParagraphWithText(note1, parText); @@ -250,6 +278,13 @@ public class LuceneSearchTest { return p; } + private Paragraph addParagraphWithTextAndTitle(Note note, String text, String title) { + Paragraph p = note.addParagraph(); + p.setText(text); + p.setTitle(title); + return p; + } + private Note newNote(String name) { Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex); note.setName(name);
