[ZEPPELIN-3593] Change LuceneSearch's directory to file system from memory
### What is this PR for?
Reducing a usage of memory by moving Lucene search's directory to the file system
### What type of PR is it?
[Improvement]
### Todos
* [x] - Replace `RamDirectory` to `MMapDirectory`
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3593
### How should this be tested?
It doesn't affect any current behaviors. Test search service
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jongyoul Lee <jongyoul@gmail.com>
Closes #3058 from jongyoul/ZEPPELIN-3593 and squashes the following commits:
cdae583c4 [Jongyoul Lee] Add a logic to check if directoryPath is null or not
5a383c4ab [Jongyoul Lee] Add `RamDirectory` to use search service in case where disk is not available
3b76cc1c0 [Jongyoul Lee] Set a default temporary directory to `java.io.tmpdir` Clean up javadoc
1e415e00e [Jongyoul Lee] Add deletion logic for the temporary directory
d26e64f88 [Jongyoul Lee] Replace `RamDirectory` to `MMapDirectory` provided by Lucene
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 635adbf..0b13e59 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -17,6 +17,12 @@
package org.apache.zeppelin.conf;
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
@@ -25,13 +31,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
* Zeppelin configuration.
*
@@ -494,10 +493,10 @@
}
public String getRelativeDir(String path) {
- if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ if (path != null && path.startsWith(File.separator) || isWindowsPath(path)) {
return path;
} else {
- return getString(ConfVars.ZEPPELIN_HOME) + "/" + path;
+ return getString(ConfVars.ZEPPELIN_HOME) + File.separator + path;
}
}
@@ -623,6 +622,10 @@
return getString(ConfVars.ZEPPELIN_PROXY_PASSWORD);
}
+ public String getZeppelinSearchTempPath() {
+ return getRelativeDir(ConfVars.ZEPPELIN_SEARCH_TEMP_PATH);
+ }
+
public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
@@ -823,7 +826,8 @@
ZEPPELIN_NOTEBOOK_CRON_FOLDERS("zeppelin.notebook.cron.folders", null),
ZEPPELIN_PROXY_URL("zeppelin.proxy.url", null),
ZEPPELIN_PROXY_USER("zeppelin.proxy.user", null),
- ZEPPELIN_PROXY_PASSWORD("zeppelin.proxy.password", null);
+ ZEPPELIN_PROXY_PASSWORD("zeppelin.proxy.password", null),
+ ZEPPELIN_SEARCH_TEMP_PATH("zeppelin.search.temp.path", System.getProperty("java.io.tmpdir"));
private String varName;
@SuppressWarnings("rawtypes")
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 9e2c5f8..5c85c23 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -168,7 +168,7 @@
notebookWsServer, notebookWsServer);
this.replFactory = new InterpreterFactory(interpreterSettingManager);
this.notebookRepo = new NotebookRepoSync(conf);
- this.noteSearchService = new LuceneSearch();
+ this.noteSearchService = new LuceneSearch(conf);
this.notebookAuthorization = NotebookAuthorization.getInstance();
this.credentials = new Credentials(
conf.credentialsPersist(),
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
index b43c453..49719a6 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/search/LuceneSearch.java
@@ -16,14 +16,20 @@
*/
package org.apache.zeppelin.search;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-
+import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -39,7 +45,6 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@@ -51,43 +56,50 @@
import org.apache.lucene.search.highlight.TextFragment;
import org.apache.lucene.search.highlight.TokenSources;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
/**
- * Search (both, indexing and query) the notebooks using Lucene.
- *
- * Query is thread-safe, as creates new IndexReader every time.
- * Index is thread-safe, as re-uses single IndexWriter, which is thread-safe.
+ * Search (both, indexing and query) the notebooks using Lucene. Query is thread-safe, as creates
+ * new IndexReader every time. Index is thread-safe, as re-uses single IndexWriter, which is
+ * thread-safe.
*/
public class LuceneSearch implements SearchService {
- private static final Logger LOG = LoggerFactory.getLogger(LuceneSearch.class);
+ private static final Logger logger = LoggerFactory.getLogger(LuceneSearch.class);
private static final String SEARCH_FIELD_TEXT = "contents";
private static final String SEARCH_FIELD_TITLE = "header";
- static final String PARAGRAPH = "paragraph";
- static final String ID_FIELD = "id";
+ private static final String PARAGRAPH = "paragraph";
+ private static final String ID_FIELD = "id";
- Directory ramDirectory;
- Analyzer analyzer;
- IndexWriterConfig iwc;
- IndexWriter writer;
+ private Directory directory;
+ private Path directoryPath;
+ private Analyzer analyzer;
+ private IndexWriterConfig indexWriterConfig;
+ private IndexWriter indexWriter;
- public LuceneSearch() {
- ramDirectory = new RAMDirectory();
- analyzer = new StandardAnalyzer();
- iwc = new IndexWriterConfig(analyzer);
+ public LuceneSearch(ZeppelinConfiguration zeppelinConfiguration) {
try {
- writer = new IndexWriter(ramDirectory, iwc);
+ this.directoryPath =
+ Files.createTempDirectory(
+ Paths.get(zeppelinConfiguration.getZeppelinSearchTempPath()), "zeppelin-search-");
+ this.directory = new MMapDirectory(directoryPath);
} catch (IOException e) {
- LOG.error("Failed to create new IndexWriter", e);
+ logger.error(
+ "Failed to create temporary directory for search service. Use memory instead", e);
+ this.directory = new RAMDirectory();
+ }
+ this.analyzer = new StandardAnalyzer();
+ this.indexWriterConfig = new IndexWriterConfig(analyzer);
+ try {
+ this.indexWriter = new IndexWriter(directory, indexWriterConfig);
+ } catch (IOException e) {
+ logger.error("Failed to create new IndexWriter", e);
}
}
@@ -96,51 +108,49 @@
*/
@Override
public List<Map<String, String>> query(String queryStr) {
- if (null == ramDirectory) {
+ if (null == directory) {
throw new IllegalStateException(
"Something went wrong on instance creation time, index dir is null");
}
List<Map<String, String>> result = Collections.emptyList();
- try (IndexReader indexReader = DirectoryReader.open(ramDirectory)) {
+ try (IndexReader indexReader = DirectoryReader.open(directory)) {
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
Analyzer analyzer = new StandardAnalyzer();
- MultiFieldQueryParser parser = new MultiFieldQueryParser(
- new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE},
- analyzer);
+ MultiFieldQueryParser parser =
+ new MultiFieldQueryParser(new String[] {SEARCH_FIELD_TEXT, SEARCH_FIELD_TITLE}, analyzer);
Query query = parser.parse(queryStr);
- LOG.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT));
+ logger.debug("Searching for: " + query.toString(SEARCH_FIELD_TEXT));
SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter();
Highlighter highlighter = new Highlighter(htmlFormatter, new QueryScorer(query));
result = doSearch(indexSearcher, query, analyzer, highlighter);
- indexReader.close();
} catch (IOException e) {
- LOG.error("Failed to open index dir {}, make sure indexing finished OK", ramDirectory, e);
+ logger.error("Failed to open index dir {}, make sure indexing finished OK", directory, e);
} catch (ParseException e) {
- LOG.error("Failed to parse query " + queryStr, e);
+ logger.error("Failed to parse query " + queryStr, e);
}
return result;
}
- private List<Map<String, String>> doSearch(IndexSearcher searcher, Query query,
- Analyzer analyzer, Highlighter highlighter) {
+ private List<Map<String, String>> doSearch(
+ IndexSearcher searcher, Query query, Analyzer analyzer, Highlighter highlighter) {
List<Map<String, String>> matchingParagraphs = Lists.newArrayList();
ScoreDoc[] hits;
try {
hits = searcher.search(query, 20).scoreDocs;
for (int i = 0; i < hits.length; i++) {
- LOG.debug("doc={} score={}", hits[i].doc, hits[i].score);
+ logger.debug("doc={} score={}", hits[i].doc, hits[i].score);
int id = hits[i].doc;
Document doc = searcher.doc(id);
String path = doc.get(ID_FIELD);
if (path != null) {
- LOG.debug((i + 1) + ". " + path);
+ logger.debug((i + 1) + ". " + path);
String title = doc.get("title");
if (title != null) {
- LOG.debug(" Title: {}", doc.get("title"));
+ logger.debug(" Title: {}", doc.get("title"));
}
String text = doc.get(SEARCH_FIELD_TEXT);
@@ -148,34 +158,38 @@
String fragment = "";
if (text != null) {
- TokenStream tokenStream = TokenSources.getTokenStream(searcher.getIndexReader(), id,
- SEARCH_FIELD_TEXT, analyzer);
+ TokenStream tokenStream =
+ TokenSources.getTokenStream(
+ searcher.getIndexReader(), id, SEARCH_FIELD_TEXT, analyzer);
TextFragment[] frag = highlighter.getBestTextFragments(tokenStream, text, true, 3);
- LOG.debug(" {} fragments found for query '{}'", frag.length, query);
+ logger.debug(" {} fragments found for query '{}'", frag.length, query);
for (int j = 0; j < frag.length; j++) {
if ((frag[j] != null) && (frag[j].getScore() > 0)) {
- LOG.debug(" Fragment: {}", frag[j].toString());
+ logger.debug(" Fragment: {}", frag[j].toString());
}
}
fragment = (frag != null && frag.length > 0) ? frag[0].toString() : "";
}
if (header != null) {
- TokenStream tokenTitle = TokenSources.getTokenStream(searcher.getIndexReader(), id,
- SEARCH_FIELD_TITLE, analyzer);
+ TokenStream tokenTitle =
+ TokenSources.getTokenStream(
+ searcher.getIndexReader(), id, SEARCH_FIELD_TITLE, analyzer);
TextFragment[] frgTitle = highlighter.getBestTextFragments(tokenTitle, header, true, 3);
header = (frgTitle != null && frgTitle.length > 0) ? frgTitle[0].toString() : "";
} else {
header = "";
}
- matchingParagraphs.add(ImmutableMap.of("id", path, // <noteId>/paragraph/<paragraphId>
- "name", title, "snippet", fragment, "text", text, "header", header));
+ matchingParagraphs.add(
+ ImmutableMap.of(
+ "id", path, // <noteId>/paragraph/<paragraphId>
+ "name", title, "snippet", fragment, "text", text, "header", header));
} else {
- LOG.info("{}. No {} for this document", i + 1, ID_FIELD);
+ logger.info("{}. No {} for this document", i + 1, ID_FIELD);
}
}
} catch (IOException | InvalidTokenOffsetsException e) {
- LOG.error("Exception on searching for {}", query, e);
+ logger.error("Exception on searching for {}", query, e);
}
return matchingParagraphs;
}
@@ -186,7 +200,7 @@
@Override
public void updateIndexDoc(Note note) throws IOException {
updateIndexNoteName(note);
- for (Paragraph p: note.getParagraphs()) {
+ for (Paragraph p : note.getParagraphs()) {
updateIndexParagraph(note, p);
}
}
@@ -194,9 +208,9 @@
private void updateIndexNoteName(Note note) throws IOException {
String noteName = note.getName();
String noteId = note.getId();
- LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName);
+ logger.debug("Indexing Notebook {}, '{}'", noteId, noteName);
if (null == noteName || noteName.isEmpty()) {
- LOG.debug("Skipping empty notebook name");
+ logger.debug("Skipping empty notebook name");
return;
}
updateDoc(noteId, noteName, null);
@@ -204,15 +218,15 @@
private void updateIndexParagraph(Note note, Paragraph p) throws IOException {
if (p.getText() == null) {
- LOG.debug("Skipping empty paragraph");
+ logger.debug("Skipping empty paragraph");
return;
}
updateDoc(note.getId(), note.getName(), p);
}
/**
- * Updates index for the given note: either note.name or a paragraph If
- * paragraph is <code>null</code> - updates only for the note.name
+ * Updates index for the given note: either note.name or a paragraph If paragraph is <code>null
+ * </code> - updates only for the note.name
*
* @param noteId
* @param noteName
@@ -223,16 +237,16 @@
String id = formatId(noteId, p);
Document doc = newDocument(id, noteName, p);
try {
- writer.updateDocument(new Term(ID_FIELD, id), doc);
- writer.commit();
+ indexWriter.updateDocument(new Term(ID_FIELD, id), doc);
+ indexWriter.commit();
} catch (IOException e) {
- LOG.error("Failed to updaet index of notebook {}", noteId, e);
+ logger.error("Failed to updaet index of notebook {}", noteId, e);
}
}
/**
- * If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>,
- * otherwise it's just <noteId>.
+ * If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>, otherwise it's just
+ * <noteId>.
*/
static String formatId(String noteId, Paragraph p) {
String id = noteId;
@@ -253,8 +267,7 @@
}
/**
- * If paragraph is not null, indexes code in the paragraph, otherwise indexes
- * the notebook name.
+ * If paragraph is not null, indexes code in the paragraph, otherwise indexes the notebook name.
*
* @param id id of the document, different for Note name and paragraph
* @param noteName name of the note
@@ -294,15 +307,17 @@
docsIndexed++;
}
} catch (IOException e) {
- LOG.error("Failed to index all Notebooks", e);
+ logger.error("Failed to index all Notebooks", e);
} finally {
try { // save what's been indexed, even if not full collection
- writer.commit();
+ indexWriter.commit();
} catch (IOException e) {
- LOG.error("Failed to save index", e);
+ logger.error("Failed to save index", e);
}
long end = System.nanoTime();
- LOG.info("Indexing {} notebooks took {}ms", docsIndexed,
+ logger.info(
+ "Indexing {} notebooks took {}ms",
+ docsIndexed,
TimeUnit.NANOSECONDS.toMillis(end - start));
}
}
@@ -314,9 +329,9 @@
public void addIndexDoc(Note note) {
try {
addIndexDocAsync(note);
- writer.commit();
+ indexWriter.commit();
} catch (IOException e) {
- LOG.error("Failed to add note {} to index", note, e);
+ logger.error("Failed to add note {} to index", note, e);
}
}
@@ -327,13 +342,13 @@
* @throws IOException
*/
private void addIndexDocAsync(Note note) throws IOException {
- indexNoteName(writer, note.getId(), note.getName());
+ indexNoteName(indexWriter, note.getId(), note.getName());
for (Paragraph doc : note.getParagraphs()) {
if (doc.getText() == null) {
- LOG.debug("Skipping empty paragraph");
+ logger.debug("Skipping empty paragraph");
continue;
}
- indexDoc(writer, note.getId(), note.getName(), doc);
+ indexDoc(indexWriter, note.getId(), note.getName(), doc);
}
}
@@ -356,18 +371,18 @@
private void deleteDoc(Note note, Paragraph p) {
if (null == note) {
- LOG.error("Trying to delete note by reference to NULL");
+ logger.error("Trying to delete note by reference to NULL");
return;
}
String fullNoteOrJustParagraph = formatDeleteId(note.getId(), p);
- LOG.debug("Deleting note {}, out of: {}", note.getId(), writer.numDocs());
+ logger.debug("Deleting note {}, out of: {}", note.getId(), indexWriter.numDocs());
try {
- writer.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph)));
- writer.commit();
+ indexWriter.deleteDocuments(new WildcardQuery(new Term(ID_FIELD, fullNoteOrJustParagraph)));
+ indexWriter.commit();
} catch (IOException e) {
- LOG.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e);
+ logger.error("Failed to delete {} from index by '{}'", note, fullNoteOrJustParagraph, e);
}
- LOG.debug("Done, index contains {} docs now" + writer.numDocs());
+ logger.debug("Done, index contains {} docs now" + indexWriter.numDocs());
}
/* (non-Javadoc)
@@ -376,9 +391,12 @@
@Override
public void close() {
try {
- writer.close();
+ indexWriter.close();
+ if (null != directoryPath) {
+ FileUtils.deleteDirectory(directoryPath.toFile());
+ }
} catch (IOException e) {
- LOG.error("Failed to .close() the notebook index", e);
+ logger.error("Failed to .close() the notebook index", e);
}
}
@@ -388,24 +406,19 @@
* @throws IOException
*/
private void indexNoteName(IndexWriter w, String noteId, String noteName) throws IOException {
- LOG.debug("Indexing Notebook {}, '{}'", noteId, noteName);
+ logger.debug("Indexing Notebook {}, '{}'", noteId, noteName);
if (null == noteName || noteName.isEmpty()) {
- LOG.debug("Skipping empty notebook name");
+ logger.debug("Skipping empty notebook name");
return;
}
indexDoc(w, noteId, noteName, null);
}
- /**
- * Indexes a single document:
- * - code of the paragraph (if non-null)
- * - or just a note name
- */
+ /** Indexes a single document: - code of the paragraph (if non-null) - or just a note name */
private void indexDoc(IndexWriter w, String noteId, String noteName, Paragraph p)
throws IOException {
String id = formatId(noteId, p);
Document doc = newDocument(id, noteName, p);
w.addDocument(doc);
}
-
}
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
index 0ac4697..aeead7e 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
@@ -17,14 +17,15 @@
package org.apache.zeppelin.search;
import static com.google.common.truth.Truth.assertThat;
-import static org.mockito.Mockito.*;
import static org.apache.zeppelin.search.LuceneSearch.formatId;
+import static org.mockito.Mockito.mock;
+import com.google.common.base.Splitter;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Note;
@@ -36,14 +37,12 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.common.base.Splitter;
-
public class LuceneSearchTest {
private static NotebookRepo notebookRepoMock;
private static InterpreterFactory interpreterFactory;
private static InterpreterSettingManager interpreterSettingManager;
-
+
private SearchService noteSearchService;
private AuthenticationInfo anonymous;
@@ -53,13 +52,13 @@
interpreterFactory = mock(InterpreterFactory.class);
interpreterSettingManager = mock(InterpreterSettingManager.class);
-// when(replLoaderMock.getInterpreterSettings())
-// .thenReturn(ImmutableList.<InterpreterSetting>of());
+ // when(replLoaderMock.getInterpreterSettings())
+ // .thenReturn(ImmutableList.<InterpreterSetting>of());
}
@Before
public void startUp() {
- noteSearchService = new LuceneSearch();
+ noteSearchService = new LuceneSearch(ZeppelinConfiguration.create());
anonymous = new AuthenticationInfo("anonymous");
}
@@ -68,42 +67,45 @@
noteSearchService.close();
}
- @Test public void canIndexNotebook() {
- //give
+ @Test
+ public void canIndexNotebook() {
+ // give
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraph("Notebook2", "not test");
List<Note> notebook = Arrays.asList(note1, note2);
- //when
+ // when
noteSearchService.addIndexDocs(notebook);
}
- @Test public void canIndexAndQuery() {
- //given
+ @Test
+ public void canIndexAndQuery() {
+ // given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- //when
+ // when
List<Map<String, String>> results = noteSearchService.query("all");
- //then
+ // then
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(1);
assertThat(results.get(0))
- .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph()));
+ .containsEntry("id", formatId(note2.getId(), note2.getLastParagraph()));
}
- @Test public void canIndexAndQueryByNotebookName() {
- //given
+ @Test
+ public void canIndexAndQueryByNotebookName() {
+ // given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- //when
+ // when
List<Map<String, String>> results = noteSearchService.query("Notebook1");
- //then
+ // then
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(1);
assertThat(results.get(0)).containsEntry("id", note1.getId());
@@ -111,15 +113,15 @@
@Test
public void canIndexAndQueryByParagraphTitle() {
- //given
+ // given
Note note1 = newNoteWithParagraph("Notebook1", "test", "testingTitleSearch");
Note note2 = newNoteWithParagraph("Notebook2", "not test", "notTestingTitleSearch");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- //when
+ // when
List<Map<String, String>> results = noteSearchService.query("testingTitleSearch");
- //then
+ // then
assertThat(results).isNotEmpty();
assertThat(results.size()).isAtLeast(1);
int TitleHits = 0;
@@ -131,41 +133,44 @@
assertThat(TitleHits).isAtLeast(1);
}
- @Test public void indexKeyContract() throws IOException {
- //give
+ @Test
+ public void indexKeyContract() throws IOException {
+ // give
Note note1 = newNoteWithParagraph("Notebook1", "test");
- //when
+ // when
noteSearchService.addIndexDoc(note1);
- //then
- String id = resultForQuery("test").get(0).get(LuceneSearch.ID_FIELD);
+ // then
+ String id = resultForQuery("test").get(0).get("id"); // LuceneSearch.ID_FIELD
- assertThat(Splitter.on("/").split(id)) //key structure <noteId>/paragraph/<paragraphId>
- .containsAllOf(note1.getId(), LuceneSearch.PARAGRAPH, note1.getLastParagraph().getId());
+ assertThat(Splitter.on("/").split(id)) // key structure <noteId>/paragraph/<paragraphId>
+ .containsAllOf(
+ note1.getId(), "paragraph", note1.getLastParagraph().getId()); // LuceneSearch.PARAGRAPH
}
- @Test //(expected=IllegalStateException.class)
+ @Test // (expected=IllegalStateException.class)
public void canNotSearchBeforeIndexing() {
- //given NO noteSearchService.index() was called
- //when
+ // given NO noteSearchService.index() was called
+ // when
List<Map<String, String>> result = noteSearchService.query("anything");
- //then
+ // then
assertThat(result).isEmpty();
- //assert logs were printed
- //"ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory"
+ // assert logs were printed
+ // "ERROR org.apache.zeppelin.search.SearchService:97 - Failed to open index dir RAMDirectory"
}
- @Test public void canIndexAndReIndex() throws IOException {
- //given
+ @Test
+ public void canIndexAndReIndex() throws IOException {
+ // given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
- //when
+ // when
Paragraph p2 = note2.getLastParagraph();
p2.setText("test indeed");
noteSearchService.updateIndexDoc(note2);
- //then
+ // then
List<Map<String, String>> results = noteSearchService.query("all");
assertThat(results).isEmpty();
@@ -173,24 +178,26 @@
assertThat(results).isNotEmpty();
}
- @Test public void canDeleteNull() throws IOException {
- //give
+ @Test
+ public void canDeleteNull() throws IOException {
+ // give
// looks like a bug in web UI: it tries to delete a note twice (after it has just been deleted)
- //when
+ // when
noteSearchService.deleteIndexDocs(null);
}
- @Test public void canDeleteFromIndex() throws IOException {
- //given
+ @Test
+ public void canDeleteFromIndex() throws IOException {
+ // given
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("Notebook2")).isNotEmpty();
- //when
+ // when
noteSearchService.deleteIndexDocs(note2);
- //then
+ // then
assertThat(noteSearchService.query("all")).isEmpty();
assertThat(resultForQuery("Notebook2")).isEmpty();
@@ -199,43 +206,46 @@
assertThat(results.size()).isEqualTo(1);
}
- @Test public void indexParagraphUpdatedOnNoteSave() throws IOException {
- //given: total 2 notebooks, 3 paragraphs
+ @Test
+ public void indexParagraphUpdatedOnNoteSave() throws IOException {
+ // given: total 2 notebooks, 3 paragraphs
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
- //when
+ // when
Paragraph p1 = note1.getLastParagraph();
p1.setText("no no no");
note1.persist(anonymous);
- //then
+ // then
assertThat(resultForQuery("Notebook1").size()).isEqualTo(1);
List<Map<String, String>> results = resultForQuery("test");
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(2);
- //does not include Notebook1's paragraph any more
- for (Map<String, String> result: results) {
- assertThat(result.get("id").startsWith(note1.getId())).isFalse();;
+ // does not include Notebook1's paragraph any more
+ for (Map<String, String> result : results) {
+ assertThat(result.get("id").startsWith(note1.getId())).isFalse();
+ ;
}
}
- @Test public void indexNoteNameUpdatedOnNoteSave() throws IOException {
- //given: total 2 notebooks, 3 paragraphs
+ @Test
+ public void indexNoteNameUpdatedOnNoteSave() throws IOException {
+ // given: total 2 notebooks, 3 paragraphs
Note note1 = newNoteWithParagraph("Notebook1", "test");
Note note2 = newNoteWithParagraphs("Notebook2", "not test", "not test at all");
noteSearchService.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
- //when
+ // when
note1.setName("NotebookN");
note1.persist(anonymous);
- //then
+ // then
assertThat(resultForQuery("Notebook1")).isEmpty();
assertThat(resultForQuery("NotebookN")).isNotEmpty();
assertThat(resultForQuery("NotebookN").size()).isEqualTo(1);
@@ -246,8 +256,7 @@
}
/**
- * Creates a new Note \w given name,
- * adds a new paragraph \w given text
+ * Creates a new Note \w given name, adds a new paragraph \w given text
*
* @param noteName name of the note
* @param parText text of the paragraph
@@ -259,16 +268,13 @@
return note1;
}
- private Note newNoteWithParagraph(String noteName, String parText,String title) {
+ private Note newNoteWithParagraph(String noteName, String parText, String title) {
Note note = newNote(noteName);
addParagraphWithTextAndTitle(note, parText, title);
return note;
}
- /**
- * Creates a new Note \w given name,
- * adds N paragraphs \w given texts
- */
+ /** Creates a new Note \w given name, adds N paragraphs \w given texts */
private Note newNoteWithParagraphs(String noteName, String... parTexts) {
Note note1 = newNote(noteName);
for (String parText : parTexts) {
@@ -291,9 +297,16 @@
}
private Note newNote(String name) {
- Note note = new Note(notebookRepoMock, interpreterFactory, interpreterSettingManager, null, noteSearchService, null, null);
+ Note note =
+ new Note(
+ notebookRepoMock,
+ interpreterFactory,
+ interpreterSettingManager,
+ null,
+ noteSearchService,
+ null,
+ null);
note.setName(name);
return note;
}
-
}
