SENTRY-989: RealTimeGet with explicit ids can bypass document level authorization (Gregory Chanan, reviewed by Hao Hao)
diff --git a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java
new file mode 100644
index 0000000..db182ef
--- /dev/null
+++ b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+
+import org.apache.solr.handler.component.RealTimeGetComponent;
+import org.apache.solr.handler.component.SecureRealTimeGetComponent;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SecureRealTimeGetHandler extends RealTimeGetHandler {
+ @Override
+ protected List<String> getDefaultComponents()
+ {
+ List<String> names = new ArrayList<>(1);
+ names.add(RealTimeGetComponent.COMPONENT_NAME);
+ names.add(SecureRealTimeGetComponent.COMPONENT_NAME);
+ return names;
+ }
+}
diff --git a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java
index 666c088..be46a85 100644
--- a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java
+++ b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java
@@ -17,6 +17,12 @@
package org.apache.solr.handler.component;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.solr.common.SolrException;
@@ -69,6 +75,40 @@
.append(value).append("}");
}
+ public String getFilterQueryStr(Set<String> roles) {
+ if (roles != null && roles.size() > 0) {
+ StringBuilder builder = new StringBuilder();
+ for (String role : roles) {
+ addRawClause(builder, authField, role);
+ }
+ if (allRolesToken != null && !allRolesToken.isEmpty()) {
+ addRawClause(builder, authField, allRolesToken);
+ }
+ return builder.toString();
+ }
+ return null;
+ }
+
+ private BooleanClause getBooleanClause(String authField, String value) {
+ Term t = new Term(authField, value);
+ return new BooleanClause(new TermQuery(t), BooleanClause.Occur.SHOULD);
+ }
+
+ public Query getFilterQuery(Set<String> roles) {
+ if (roles != null && roles.size() > 0) {
+ BooleanQuery query = new BooleanQuery();
+ for (String role : roles) {
+ query.add(getBooleanClause(authField, role));
+ }
+ if (allRolesToken != null && !allRolesToken.isEmpty()) {
+ query.add(getBooleanClause(authField, allRolesToken));
+ }
+ return query;
+ }
+
+ return null;
+ }
+
@Override
public void prepare(ResponseBuilder rb) throws IOException {
if (!enabled) {
@@ -82,16 +122,9 @@
}
Set<String> roles = sentryInstance.getRoles(userName);
if (roles != null && roles.size() > 0) {
- StringBuilder builder = new StringBuilder();
- for (String role : roles) {
- addRawClause(builder, authField, role);
- }
- if (allRolesToken != null && !allRolesToken.isEmpty()) {
- addRawClause(builder, authField, allRolesToken);
- }
+ String filterQuery = getFilterQueryStr(roles);
ModifiableSolrParams newParams = new ModifiableSolrParams(rb.req.getParams());
- String result = builder.toString();
- newParams.add("fq", result);
+ newParams.add("fq", filterQuery);
rb.req.setParams(newParams);
} else {
throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED,
@@ -113,4 +146,8 @@
public String getSource() {
return "$URL$";
}
+
+ public boolean getEnabled() {
+ return enabled;
+ }
}
diff --git a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java
new file mode 100644
index 0000000..e692f54
--- /dev/null
+++ b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.component;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.BytesRef;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.response.transform.DocTransformer;
+import org.apache.solr.response.transform.DocTransformers;
+import org.apache.solr.response.transform.TransformContext;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SolrReturnFields;
+import org.apache.solr.search.ReturnFields;
+import org.apache.solr.sentry.SentryIndexAuthorizationSingleton;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.UpdateCommand;
+import org.apache.solr.update.UpdateLog;
+import org.apache.solr.util.RefCounted;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+public class SecureRealTimeGetComponent extends SearchComponent
+{
+ private static Logger log =
+ LoggerFactory.getLogger(SecureRealTimeGetComponent.class);
+ public static String ID_FIELD_NAME = "_reserved_sentry_id";
+ public static final String COMPONENT_NAME = "secureGet";
+
+ private SentryIndexAuthorizationSingleton sentryInstance;
+
+ public SecureRealTimeGetComponent() {
+ this(SentryIndexAuthorizationSingleton.getInstance());
+ }
+
+ @VisibleForTesting
+ public SecureRealTimeGetComponent(SentryIndexAuthorizationSingleton sentryInstance) {
+ super();
+ this.sentryInstance = sentryInstance;
+ }
+
+ @Override
+ public void prepare(ResponseBuilder rb) throws IOException {
+ QueryDocAuthorizationComponent docComponent =
+ (QueryDocAuthorizationComponent)rb.req.getCore().getSearchComponent("queryDocAuthorization");
+ if (docComponent != null) {
+ String userName = sentryInstance.getUserName(rb.req);
+ String superUser = (System.getProperty("solr.authorization.superuser", "solr"));
+ // security is never applied to the super user; for example, if solr internally is using
+ // real time get for replica synchronization, we need to return all the documents.
+ if (docComponent.getEnabled() && !superUser.equals(userName)) {
+ Set<String> roles = sentryInstance.getRoles(userName);
+ if (roles != null && roles.size() > 0) {
+ SolrReturnFields savedReturnFields = (SolrReturnFields)rb.rsp.getReturnFields();
+ if (savedReturnFields == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Not able to authorize request because ReturnFields is invalid: " + savedReturnFields);
+ }
+ DocTransformer savedTransformer = savedReturnFields.getTransformer();
+ Query filterQuery = docComponent.getFilterQuery(roles);
+ if (filterQuery != null) {
+ SolrReturnFields solrReturnFields = new AddDocIdReturnFields(rb.req, savedTransformer, filterQuery);
+ rb.rsp.setReturnFields(solrReturnFields);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED,
+ "Request from user: " + userName +
+ "rejected because filter query was unable to be generated");
+ }
+ } else {
+ throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED,
+ "Request from user: " + userName +
+ " rejected because user is not associated with any roles");
+ }
+ }
+ } else {
+ throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED,
+ "RealTimeGetRequest request " +
+ " rejected because \"queryDocAuthorization\" component not defined");
+ }
+ }
+
+ @Override
+ public void process(ResponseBuilder rb) throws IOException {
+ if (!(rb.rsp.getReturnFields() instanceof AddDocIdReturnFields)) {
+ log.info("Skipping application of SecureRealTimeGetComponent because "
+ + " return field wasn't applied in prepare phase");
+ return;
+ }
+
+ final SolrQueryResponse rsp = rb.rsp;
+ ResponseFormatDocs responseFormatDocs = getResponseFormatDocs(rsp);
+ if (responseFormatDocs == null) {
+ return; // no documents to check
+ }
+ final SolrDocumentList docList = responseFormatDocs.getDocList();
+ final AddDocIdReturnFields addDocIdRf = (AddDocIdReturnFields)rb.rsp.getReturnFields();
+ final Query filterQuery = addDocIdRf.getFilterQuery();
+ final DocTransformer transformer = addDocIdRf.getOriginalTransformer();
+
+ // we replaced the original transfer in order to add the document id, reapply it here
+ // so return documents in the correct format.
+ if (transformer != null) {
+ TransformContext context = new TransformContext();
+ context.req = rb.req;
+ transformer.setContext(context);
+ }
+
+ SolrCore core = rb.req.getCore();
+ UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
+ SchemaField idField = core.getLatestSchema().getUniqueKeyField();
+ FieldType fieldType = idField.getType();
+ boolean openedRealTimeSearcher = false;
+ RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher();
+
+ SolrDocumentList docListToReturn = new SolrDocumentList();
+ try {
+ SolrIndexSearcher searcher = searcherHolder.get();
+ for (SolrDocument doc : docList) {
+ // -1 doc id indicates this value was read from log; we need to open
+ // a new real time searcher to run the filter query against
+ if (doc.get(ID_FIELD_NAME) == -1 && !openedRealTimeSearcher) {
+ searcherHolder.decref();
+ // hack to clear ulog maps since we don't have
+ // openRealtimeSearcher API from SOLR-8436
+ AddUpdateCommand cmd = new AddUpdateCommand(rb.req);
+ cmd.setFlags(UpdateCommand.REPLAY);
+ ulog.add(cmd, true);
+
+ searcherHolder = core.getRealtimeSearcher();
+ searcher = searcherHolder.get();
+ openedRealTimeSearcher = true;
+ }
+
+ int docid = getFilteredInternalDocId(doc, idField, fieldType, filterQuery, searcher);
+ if (docid < 0) continue;
+ Document luceneDocument = searcher.doc(docid);
+ SolrDocument newDoc = toSolrDoc(luceneDocument, core.getLatestSchema());
+ if( transformer != null ) {
+ transformer.transform(newDoc, docid);
+ }
+ docListToReturn.add(newDoc);
+ }
+ } finally {
+ searcherHolder.decref();
+ searcherHolder = null;
+ }
+ if (responseFormatDocs.getUseResponseField()) {
+ rsp.getValues().remove("response");
+ docListToReturn.setNumFound(docListToReturn.size());
+ rsp.add("response", docListToReturn);
+ } else {
+ rsp.getValues().remove("doc");
+ rsp.add("doc", docListToReturn.size() > 0 ? docListToReturn.get(0) : null);
+ }
+ }
+
+ private static SolrDocument toSolrDoc(Document doc, IndexSchema schema) {
+ SolrDocument out = new SolrDocument();
+ for ( IndexableField f : doc.getFields() ) {
+ // Make sure multivalued fields are represented as lists
+ Object existing = out.get(f.name());
+ if (existing == null) {
+ SchemaField sf = schema.getFieldOrNull(f.name());
+
+ // don't return copyField targets
+ if (sf != null && schema.isCopyFieldTarget(sf)) continue;
+
+ if (sf != null && sf.multiValued()) {
+ List<Object> vals = new ArrayList<>();
+ vals.add( f );
+ out.setField( f.name(), vals );
+ }
+ else{
+ out.setField( f.name(), f );
+ }
+ }
+ else {
+ out.addField( f.name(), f );
+ }
+ }
+ return out;
+ }
+
+ // get the response format to use and the documents to check
+ private static ResponseFormatDocs getResponseFormatDocs(final SolrQueryResponse rsp) {
+ SolrDocumentList docList = (SolrDocumentList)rsp.getValues().get("response");
+ SolrDocument singleDoc = (SolrDocument)rsp.getValues().get("doc");
+ if (docList == null && singleDoc == null) {
+ return null; // no documents to filter
+ }
+ if (docList != null && singleDoc != null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Not able to filter secure reponse, RealTimeGet returned both a doc list and " +
+ "an individual document");
+ }
+ final boolean useResponseField = docList != null;
+ if (docList == null) {
+ docList = new SolrDocumentList();
+ docList.add(singleDoc);
+ }
+ return new ResponseFormatDocs(useResponseField, docList);
+ }
+
+ /**
+ * @param doc SolrDocument to check
+ * @param idField field where the id is stored
+ * @param fieldType type of id field
+ * @param filterQuery Query to filter by
+ * @param searcher SolrIndexSearcher on which to apply the filter query
+ * @returns the internal docid, or -1 if doc is not found or doesn't match filter
+ */
+ private static int getFilteredInternalDocId(SolrDocument doc, SchemaField idField, FieldType fieldType,
+ Query filterQuery, SolrIndexSearcher searcher) throws IOException {
+ int docid = -1;
+ Field f = (Field)doc.getFieldValue(idField.getName());
+ String idStr = f.stringValue();
+ BytesRef idBytes = new BytesRef();
+ fieldType.readableToIndexed(idStr, idBytes);
+ // get the internal document id
+ long segAndId = searcher.lookupId(idBytes);
+
+ // if docid is valid, run it through the filter
+ if (segAndId >= 0) {
+ int segid = (int) segAndId;
+ AtomicReaderContext ctx = searcher.getTopReaderContext().leaves().get((int) (segAndId >> 32));
+ docid = segid + ctx.docBase;
+ Weight weight = filterQuery.createWeight(searcher);
+ Scorer scorer = weight.scorer(ctx, null);
+ if (scorer == null || segid != scorer.advance(segid)) {
+ // filter doesn't match.
+ docid = -1;
+ }
+ }
+ return docid;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Handle Query Document Authorization for RealTimeGet";
+ }
+
+ @Override
+ public String getSource() {
+ return "$URL$";
+ }
+
+ private static class ResponseFormatDocs {
+ private boolean useResponseField;
+ private SolrDocumentList docList;
+
+ public ResponseFormatDocs(boolean useResponseField, SolrDocumentList docList) {
+ this.useResponseField = useResponseField;
+ this.docList = docList;
+ }
+
+ public boolean getUseResponseField() { return useResponseField; }
+ public SolrDocumentList getDocList() { return docList; }
+ }
+
+ // ReturnField that adds a transformer to store the document id
+ private static class AddDocIdReturnFields extends SolrReturnFields {
+ private DocTransformer transformer;
+ private DocTransformer originalTransformer;
+ private Query filterQuery;
+
+ public AddDocIdReturnFields(SolrQueryRequest req, DocTransformer docTransformer,
+ Query filterQuery) {
+ super(req);
+ this.originalTransformer = docTransformer;
+ this.filterQuery = filterQuery;
+ final DocTransformers docTransformers = new DocTransformers();
+ if (originalTransformer != null) docTransformers.addTransformer(originalTransformer);
+ docTransformers.addTransformer(new DocIdAugmenter(ID_FIELD_NAME));
+ this.transformer = docTransformers;
+ }
+
+ @Override
+ public DocTransformer getTransformer() {
+ return transformer;
+ }
+
+ public DocTransformer getOriginalTransformer() {
+ return originalTransformer;
+ }
+
+ public Query getFilterQuery() {
+ return filterQuery;
+ }
+ }
+
+ // the Solr DocIdAugmenterFactory does not store negative doc ids;
+ // we do here.
+ private static class DocIdAugmenter extends DocTransformer
+ {
+ final String name;
+
+ public DocIdAugmenter( String display )
+ {
+ this.name = display;
+ }
+
+ @Override
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public void transform(SolrDocument doc, int docid) {
+ doc.setField( name, docid );
+ }
+ }
+
+}
diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
index 2495a9e..3a2104a 100644
--- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
+++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
@@ -28,6 +28,7 @@
import java.net.MalformedURLException;
import java.net.URI;
import java.util.Comparator;
+import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -61,6 +62,7 @@
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -90,6 +92,7 @@
protected static final Random RANDOM = new Random();
protected static final String RESOURCES_DIR = "target" + File.separator + "test-classes" + File.separator + "solr";
protected static final String CONF_DIR_IN_ZK = "conf1";
+ protected static final String DEFAULT_COLLECTION = "collection1";
protected static final int NUM_SERVERS = 4;
private static void addPropertyToSentry(StringBuilder builder, String name, String value) {
@@ -413,17 +416,30 @@
* @param solrUserName - User authenticated into Solr
* @param adminOp - Admin operation to be performed
* @param collectionName - Name of the collection to be queried
- * @param ignoreError - boolean to specify whether to ignore the error if any occurred.
- * (We may need this attribute for running DELETE command on a collection which doesn't exist)
* @throws Exception
*/
protected void verifyCollectionAdminOpPass(String solrUserName,
CollectionAction adminOp,
String collectionName) throws Exception {
+ verifyCollectionAdminOpPass(solrUserName, adminOp, collectionName, null);
+ }
+
+ /**
+ * Method to validate collection Admin operation pass
+ * @param solrUserName - User authenticated into Solr
+ * @param adminOp - Admin operation to be performed
+ * @param collectionName - Name of the collection to be queried
+ * @param params - SolrParams to use
+ * @throws Exception
+ */
+ protected void verifyCollectionAdminOpPass(String solrUserName,
+ CollectionAction adminOp,
+ String collectionName,
+ SolrParams params) throws Exception {
String originalUser = getAuthenticatedUser();
try {
setAuthenticationUser(solrUserName);
- QueryRequest request = populateCollectionAdminParams(adminOp, collectionName);
+ QueryRequest request = populateCollectionAdminParams(adminOp, collectionName, params);
CloudSolrServer solrServer = createNewCloudSolrServer();
try {
NamedList<Object> result = solrServer.request(request);
@@ -449,12 +465,27 @@
protected void verifyCollectionAdminOpFail(String solrUserName,
CollectionAction adminOp,
String collectionName) throws Exception {
+ verifyCollectionAdminOpFail(solrUserName, adminOp, collectionName, null);
+ }
+
+ /**
+ * Method to validate collection Admin operation fail
+ * @param solrUserName - User authenticated into Solr
+ * @param adminOp - Admin operation to be performed
+ * @param collectionName - Name of the collection to be queried
+ * @param params - SolrParams to use
+ * @throws Exception
+ */
+ protected void verifyCollectionAdminOpFail(String solrUserName,
+ CollectionAction adminOp,
+ String collectionName,
+ SolrParams params) throws Exception {
String originalUser = getAuthenticatedUser();
try {
setAuthenticationUser(solrUserName);
try {
- QueryRequest request = populateCollectionAdminParams(adminOp, collectionName);
+ QueryRequest request = populateCollectionAdminParams(adminOp, collectionName, params);
CloudSolrServer solrServer = createNewCloudSolrServer();
try {
NamedList<Object> result = solrServer.request(request);
@@ -483,7 +514,20 @@
* @return - instance of QueryRequest.
*/
public QueryRequest populateCollectionAdminParams(CollectionAction adminOp,
- String collectionName) {
+ String collectionName) {
+ return populateCollectionAdminParams(adminOp, collectionName, null);
+ }
+
+ /**
+ * Method to populate the Solr params based on the collection admin being performed.
+ * @param adminOp - Collection admin operation
+ * @param collectionName - Name of the collection
+ * @param params - SolrParams to use
+ * @return - instance of QueryRequest.
+ */
+ public QueryRequest populateCollectionAdminParams(CollectionAction adminOp,
+ String collectionName,
+ SolrParams params) {
ModifiableSolrParams modParams = new ModifiableSolrParams();
modParams.set(CoreAdminParams.ACTION, adminOp.name());
switch (adminOp) {
@@ -519,6 +563,14 @@
throw new IllegalArgumentException("Admin operation: " + adminOp + " is not supported!");
}
+ if (params != null) {
+ Iterator<String> it = params.getParameterNamesIterator();
+ while (it.hasNext()) {
+ String param = it.next();
+ String [] value = params.getParams(param);
+ modParams.set(param, value);
+ }
+ }
QueryRequest request = new QueryRequest(modParams);
request.setPath("/admin/collections");
return request;
@@ -701,16 +753,22 @@
}
protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception {
+ uploadConfigDirToZk(collectionConfigDir, CONF_DIR_IN_ZK);
+ }
+
+ protected void uploadConfigDirToZk(String collectionConfigDir, String confDirInZk) throws Exception {
ZkController zkController = getZkController();
- // conf1 is the config used by AbstractFullDistribZkTestBase
- zkController.uploadConfigDir(new File(collectionConfigDir),
- CONF_DIR_IN_ZK);
+ zkController.uploadConfigDir(new File(collectionConfigDir), confDirInZk);
}
protected void uploadConfigFileToZk(String file, String nameInZk) throws Exception {
+ uploadConfigFileToZk(file, nameInZk, CONF_DIR_IN_ZK);
+ }
+
+ protected void uploadConfigFileToZk(String file, String nameInZk, String confDirInZk) throws Exception {
ZkController zkController = getZkController();
zkController.getZkClient().makePath(ZkController.CONFIGS_ZKNODE + "/"
- + CONF_DIR_IN_ZK + "/" + nameInZk, new File(file), false, true);
+ + confDirInZk + "/" + nameInZk, new File(file), false, true);
}
protected CloudSolrServer createNewCloudSolrServer() throws Exception {
diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java
new file mode 100644
index 0000000..30afd4c
--- /dev/null
+++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.tests.e2e.solr;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+
+import java.util.ArrayList;
+
+public class DocLevelGenerator {
+ private String collection;
+ private String authField;
+
+ public DocLevelGenerator(String collection, String authField) {
+ this.collection = collection;
+ this.authField = authField;
+ }
+
+ /**
+ * Generates docs according to the following parameters:
+ *
+ * @param server SolrServer to use
+ * @param numDocs number of documents to generate
+ * @param evenDocsToken every even number doc gets this token added to the authField
+ * @param oddDocsToken every odd number doc gets this token added to the authField
+ * @param extraAuthFieldsCount generates this number of bogus entries in the authField
+ */
+ public void generateDocs(CloudSolrServer server, int numDocs, String evenDocsToken, String oddDocsToken, int extraAuthFieldsCount) throws Exception {
+
+ // create documents
+ ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
+ for (int i = 0; i < numDocs; ++i) {
+ SolrInputDocument doc = new SolrInputDocument();
+ String iStr = Long.toString(i);
+ doc.addField("id", iStr);
+ doc.addField("description", "description" + iStr);
+
+ // put some bogus tokens in
+ for (int k = 0; k < extraAuthFieldsCount; ++k) {
+ doc.addField(authField, authField + Long.toString(k));
+ }
+ // even docs get evenDocsToken, odd docs get oddDocsToken
+ if (i % 2 == 0) {
+ doc.addField(authField, evenDocsToken);
+ } else {
+ doc.addField(authField, oddDocsToken);
+ }
+ // add a token to all docs so we can check that we can get all
+ // documents returned
+ doc.addField(authField, "docLevel_role");
+
+ docs.add(doc);
+ }
+
+ server.add(docs);
+ server.commit(true, true);
+ }
+}
diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java
index ff508e1..46399df 100644
--- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java
+++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java
@@ -25,11 +25,15 @@
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import java.io.File;
import java.net.URLEncoder;
@@ -44,7 +48,6 @@
public class TestDocLevelOperations extends AbstractSolrSentryTestBase {
private static final Logger LOG = LoggerFactory
.getLogger(TestDocLevelOperations.class);
- private static final String DEFAULT_COLLECTION = "collection1";
private static final String AUTH_FIELD = "sentry_auth";
private static final int NUM_DOCS = 100;
private static final int EXTRA_AUTH_FIELDS = 2;
@@ -70,6 +73,31 @@
setupCollection(name);
}
+ private QueryRequest getRealTimeGetRequest() {
+ // real time get request
+ StringBuilder idsBuilder = new StringBuilder("0");
+ for (int i = 1; i < NUM_DOCS; ++i) {
+ idsBuilder.append("," + i);
+ }
+ return getRealTimeGetRequest(idsBuilder.toString());
+ }
+
+ private QueryRequest getRealTimeGetRequest(String ids) {
+ final ModifiableSolrParams idsParams = new ModifiableSolrParams();
+ idsParams.add("ids", ids);
+ return new QueryRequest() {
+ @Override
+ public String getPath() {
+ return "/get";
+ }
+
+ @Override
+ public SolrParams getParams() {
+ return idsParams;
+ }
+ };
+ }
+
/**
* Creates docs as follows and verifies queries work as expected:
* - creates NUM_DOCS documents, where the document id equals the order
@@ -84,67 +112,45 @@
// ensure no current documents
verifyDeletedocsPass(ADMIN_USER, collectionName, true);
- // create documents
- ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
- for (int i = 0; i < NUM_DOCS; ++i) {
- SolrInputDocument doc = new SolrInputDocument();
- String iStr = Long.toString(i);
- doc.addField("id", iStr);
- doc.addField("description", "description" + iStr);
-
- // put some bogus tokens in
- for (int k = 0; k < EXTRA_AUTH_FIELDS; ++k) {
- doc.addField(AUTH_FIELD, AUTH_FIELD + Long.toString(k));
- }
- // 50% of docs get "junit", 50% get "admin" as token
- if (i % 2 == 0) {
- doc.addField(AUTH_FIELD, "junit_role");
- } else {
- doc.addField(AUTH_FIELD, "admin_role");
- }
- // add a token to all docs so we can check that we can get all
- // documents returned
- doc.addField(AUTH_FIELD, "docLevel_role");
-
- docs.add(doc);
- }
CloudSolrServer server = getCloudSolrServer(collectionName);
try {
- server.add(docs);
- server.commit(true, true);
+ DocLevelGenerator generator = new DocLevelGenerator(collectionName, AUTH_FIELD);
+ generator.generateDocs(server, NUM_DOCS, "junit_role", "admin_role", EXTRA_AUTH_FIELDS);
- // queries
- SolrQuery query = new SolrQuery();
- query.setQuery("*:*");
+ querySimple(new QueryRequest(new SolrQuery("*:*")), server, checkNonAdminUsers);
+ querySimple(getRealTimeGetRequest(), server, checkNonAdminUsers);
+ } finally {
+ server.shutdown();
+ }
+ }
- // as admin -- should get the other half
- setAuthenticationUser("admin");
- QueryResponse rsp = server.query(query);
- SolrDocumentList docList = rsp.getResults();
+ private void querySimple(QueryRequest request, CloudSolrServer server,
+ boolean checkNonAdminUsers) throws Exception {
+ // as admin -- should get the other half
+ setAuthenticationUser("admin");
+ QueryResponse rsp = request.process(server);
+ SolrDocumentList docList = rsp.getResults();
+ assertEquals(NUM_DOCS / 2, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ assertEquals(1, Long.valueOf(id) % 2);
+ }
+
+ if (checkNonAdminUsers) {
+ // as junit -- should get half the documents
+ setAuthenticationUser("junit");
+ rsp = request.process(server);
+ docList = rsp.getResults();
assertEquals(NUM_DOCS / 2, docList.getNumFound());
for (SolrDocument doc : docList) {
String id = doc.getFieldValue("id").toString();
- assertEquals(1, Long.valueOf(id) % 2);
+ assertEquals(0, Long.valueOf(id) % 2);
}
- if (checkNonAdminUsers) {
- // as junit -- should get half the documents
- setAuthenticationUser("junit");
- rsp = server.query(query);
- docList = rsp.getResults();
- assertEquals(NUM_DOCS / 2, docList.getNumFound());
- for (SolrDocument doc : docList) {
- String id = doc.getFieldValue("id").toString();
- assertEquals(0, Long.valueOf(id) % 2);
- }
-
- // as docLevel -- should get all
- setAuthenticationUser("docLevel");
- rsp = server.query(query);
- assertEquals(NUM_DOCS, rsp.getResults().getNumFound());
- }
- } finally {
- server.shutdown();
+ // as docLevel -- should get all
+ setAuthenticationUser("docLevel");
+ rsp = request.process(server);
+ assertEquals(NUM_DOCS, rsp.getResults().getNumFound());
}
}
@@ -237,31 +243,10 @@
server.add(docs);
server.commit(true, true);
- // queries
- SolrQuery query = new SolrQuery();
- query.setQuery("*:*");
-
- // as admin -- should only get all roles token documents
- setAuthenticationUser("admin");
- QueryResponse rsp = server.query(query);
- SolrDocumentList docList = rsp.getResults();
- assertEquals(totalAllRolesAdded, docList.getNumFound());
- for (SolrDocument doc : docList) {
- String id = doc.getFieldValue("id").toString();
- assertEquals(0, Long.valueOf(id) % allRolesFactor);
- }
-
- // as junit -- should get junit added + onlyAllRolesAdded
- setAuthenticationUser("junit");
- rsp = server.query(query);
- docList = rsp.getResults();
- assertEquals(totalJunitAdded + totalOnlyAllRolesAdded, docList.getNumFound());
- for (SolrDocument doc : docList) {
- String id = doc.getFieldValue("id").toString();
- boolean addedJunit = (Long.valueOf(id) % junitFactor) == 0;
- boolean onlyAllRoles = !addedJunit && (Long.valueOf(id) % allRolesFactor) == 0;
- assertEquals(true, addedJunit || onlyAllRoles);
- }
+ checkAllRolesToken(new QueryRequest(new SolrQuery("*:*")), server,
+ totalAllRolesAdded, totalOnlyAllRolesAdded, allRolesFactor, totalJunitAdded, junitFactor);
+ checkAllRolesToken(getRealTimeGetRequest(), server,
+ totalAllRolesAdded, totalOnlyAllRolesAdded, allRolesFactor, totalJunitAdded, junitFactor);
} finally {
server.shutdown();
}
@@ -270,6 +255,31 @@
}
}
+ private void checkAllRolesToken(QueryRequest request, CloudSolrServer server,
+ int totalAllRolesAdded, int totalOnlyAllRolesAdded, int allRolesFactor, int totalJunitAdded, int junitFactor) throws Exception {
+ // as admin -- should only get all roles token documents
+ setAuthenticationUser("admin");
+ QueryResponse rsp = request.process(server);
+ SolrDocumentList docList = rsp.getResults();
+ assertEquals(totalAllRolesAdded, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ assertEquals(0, Long.valueOf(id) % allRolesFactor);
+ }
+
+ // as junit -- should get junit added + onlyAllRolesAdded
+ setAuthenticationUser("junit");
+ rsp = request.process(server);
+ docList = rsp.getResults();
+ assertEquals(totalJunitAdded + totalOnlyAllRolesAdded, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ boolean addedJunit = (Long.valueOf(id) % junitFactor) == 0;
+ boolean onlyAllRoles = !addedJunit && (Long.valueOf(id) % allRolesFactor) == 0;
+ assertEquals(true, addedJunit || onlyAllRoles);
+ }
+ }
+
/**
* delete the docs as "deleteUser" using deleteByQuery "deleteQueryStr".
* Verify that number of docs returned for "queryUser" equals
@@ -280,32 +290,35 @@
createDocsAndQuerySimple(collectionName, true);
CloudSolrServer server = getCloudSolrServer(collectionName);
try {
- SolrQuery query = new SolrQuery();
- query.setQuery("*:*");
-
setAuthenticationUser(deleteUser);
server.deleteByQuery(deleteByQueryStr);
server.commit();
- QueryResponse rsp = server.query(query);
- long junitResults = rsp.getResults().getNumFound();
- assertEquals(0, junitResults);
- setAuthenticationUser(queryUser);
- rsp = server.query(query);
- long docLevelResults = rsp.getResults().getNumFound();
- assertEquals(expectedQueryDocs, docLevelResults);
+ checkDeleteByQuery(new QueryRequest(new SolrQuery("*:*")), server,
+ queryUser, expectedQueryDocs);
+ checkDeleteByQuery(getRealTimeGetRequest(), server,
+ queryUser, expectedQueryDocs);
} finally {
server.shutdown();
}
}
+ private void checkDeleteByQuery(QueryRequest query, CloudSolrServer server,
+ String queryUser, int expectedQueryDocs) throws Exception {
+ QueryResponse rsp = query.process(server);
+ long junitResults = rsp.getResults().getNumFound();
+ assertEquals(0, junitResults);
+
+ setAuthenticationUser(queryUser);
+ rsp = query.process(server);
+ long docLevelResults = rsp.getResults().getNumFound();
+ assertEquals(expectedQueryDocs, docLevelResults);
+ }
+
private void deleteByIdTest(String collectionName) throws Exception {
createDocsAndQuerySimple(collectionName, true);
CloudSolrServer server = getCloudSolrServer(collectionName);
try {
- SolrQuery query = new SolrQuery();
- query.setQuery("*:*");
-
setAuthenticationUser("junit");
List<String> allIds = new ArrayList<String>(NUM_DOCS);
for (int i = 0; i < NUM_DOCS; ++i) {
@@ -314,19 +327,25 @@
server.deleteById(allIds);
server.commit();
- QueryResponse rsp = server.query(query);
- long junitResults = rsp.getResults().getNumFound();
- assertEquals(0, junitResults);
-
- setAuthenticationUser("docLevel");
- rsp = server.query(query);
- long docLevelResults = rsp.getResults().getNumFound();
- assertEquals(0, docLevelResults);
+ checkDeleteById(new QueryRequest(new SolrQuery("*:*")), server);
+ checkDeleteById(getRealTimeGetRequest(), server);
} finally {
server.shutdown();
}
}
+ private void checkDeleteById(QueryRequest request, CloudSolrServer server)
+ throws Exception {
+ QueryResponse rsp = request.process(server);
+ long junitResults = rsp.getResults().getNumFound();
+ assertEquals(0, junitResults);
+
+ setAuthenticationUser("docLevel");
+ rsp = request.process(server);
+ long docLevelResults = rsp.getResults().getNumFound();
+ assertEquals(0, docLevelResults);
+ }
+
private void updateDocsTest(String collectionName) throws Exception {
createDocsAndQuerySimple(collectionName, true);
CloudSolrServer server = getCloudSolrServer(collectionName);
@@ -335,10 +354,10 @@
String docIdStr = Long.toString(1);
// verify we can't view one of the odd documents
- SolrQuery query = new SolrQuery();
- query.setQuery("id:"+docIdStr);
- QueryResponse rsp = server.query(query);
- assertEquals(0, rsp.getResults().getNumFound());
+ QueryRequest query = new QueryRequest(new SolrQuery("id:"+docIdStr));
+ QueryRequest rtgQuery = getRealTimeGetRequest(docIdStr);
+ checkUpdateDocsQuery(query, server, 0);
+ checkUpdateDocsQuery(rtgQuery, server, 0);
// overwrite the document that we can't see
ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
@@ -351,13 +370,19 @@
server.commit();
// verify we can now view the document
- rsp = server.query(query);
- assertEquals(1, rsp.getResults().getNumFound());
+ checkUpdateDocsQuery(query, server, 1);
+ checkUpdateDocsQuery(rtgQuery, server, 1);
} finally {
server.shutdown();
}
}
+ private void checkUpdateDocsQuery(QueryRequest request, CloudSolrServer server, int expectedDocs)
+ throws Exception {
+ QueryResponse rsp = request.process(server);
+ assertEquals(expectedDocs, rsp.getResults().getNumFound());
+ }
+
@Test
public void testUpdateDeleteOperations() throws Exception {
String collectionName = "testUpdateDeleteOperations";
diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java
new file mode 100644
index 0000000..0d25562
--- /dev/null
+++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.tests.e2e.solr;
+
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+public class TestRealTimeGet extends AbstractSolrSentryTestBase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestRealTimeGet.class);
+ private static final String AUTH_FIELD = "sentry_auth";
+ private static final Random rand = new Random();
+ private String userName = null;
+
+ @Before
+ public void beforeTest() throws Exception {
+ userName = getAuthenticatedUser();
+ }
+
+ @After
+ public void afterTest() throws Exception {
+ setAuthenticationUser(userName);
+ }
+
+ private void setupCollectionWithDocSecurity(String name) throws Exception {
+ setupCollectionWithDocSecurity(name, 2);
+ }
+
+ private void setupCollectionWithDocSecurity(String name, int shards) throws Exception {
+ String configDir = RESOURCES_DIR + File.separator + DEFAULT_COLLECTION
+ + File.separator + "conf";
+ uploadConfigDirToZk(configDir, name);
+ // replace solrconfig.xml with solrconfig-doc-level.xml
+ uploadConfigFileToZk(configDir + File.separator + "solrconfig-doclevel.xml",
+ "solrconfig.xml", name);
+ ModifiableSolrParams modParams = new ModifiableSolrParams();
+ modParams.set("numShards", shards);
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < shards; ++i) {
+ if (i != 0) builder.append(",");
+ builder.append("shard").append(i+1);
+ }
+ modParams.set("shards", builder.toString());
+ verifyCollectionAdminOpPass(ADMIN_USER, CollectionAction.CREATE, name, modParams);
+ }
+
+ private void setupCollectionWithoutDocSecurity(String name) throws Exception {
+ String configDir = RESOURCES_DIR + File.separator + DEFAULT_COLLECTION
+ + File.separator + "conf";
+ uploadConfigDirToZk(configDir, name);
+ setupCollection(name);
+ }
+
+ private QueryRequest getRealTimeGetRequest(final SolrParams params) {
+ return new QueryRequest() {
+ @Override
+ public String getPath() {
+ return "/get";
+ }
+
+ @Override
+ public SolrParams getParams() {
+ return params;
+ }
+ };
+ }
+
+ private void assertExpected(ExpectedResult expectedResult, QueryResponse rsp,
+ ExpectedResult controlExpectedResult, QueryResponse controlRsp) throws Exception {
+ SolrDocumentList docList = rsp.getResults();
+ SolrDocumentList controlDocList = controlRsp.getResults();
+ SolrDocument doc = (SolrDocument)rsp.getResponse().get("doc");
+ SolrDocument controlDoc = (SolrDocument)controlRsp.getResponse().get("doc");
+
+ if (expectedResult.expectedDocs == 0) {
+ // could be null rather than 0 size, check against control that format is identical
+ assertNull("Should be no doc present: " + doc, doc);
+ assertNull("Should be no doc present: " + controlDoc, controlDoc);
+ assertTrue((docList == null && controlDocList == null) ||
+ (controlDocList.getNumFound() == 0 && controlDocList.getNumFound() == 0));
+ } else {
+ if (docList == null) {
+ assertNull(controlDocList);
+ assertNotNull(doc);
+ assertNotNull(controlDoc);
+ } else {
+ assertNotNull(controlDocList);
+ assertNull(doc);
+ assertNull(controlDoc);
+ assertEquals(expectedResult.expectedDocs, docList.getNumFound());
+ assertEquals(docList.getNumFound(), controlDocList.getNumFound());
+ }
+ }
+ }
+
+ private QueryResponse getIdResponse(ExpectedResult expectedResult) throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ for (int i = 0; i < expectedResult.ids.length; ++i) {
+ params.add("id", expectedResult.ids[ i ]);
+ }
+ if (expectedResult.fl != null) {
+ params.add("fl", expectedResult.fl);
+ }
+ QueryRequest request = getRealTimeGetRequest(params);
+ return request.process(expectedResult.server);
+ }
+
+ private QueryResponse getIdsResponse(ExpectedResult expectedResult) throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < expectedResult.ids.length; ++i) {
+ if (i != 0) builder.append(",");
+ builder.append(expectedResult.ids[ i ]);
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("ids", builder.toString());
+ if (expectedResult.fl != null) {
+ params.add("fl", expectedResult.fl);
+ }
+ QueryRequest request = getRealTimeGetRequest(params);
+ return request.process(expectedResult.server);
+ }
+
+ private void assertIdVsIds(ExpectedResult expectedResult, ExpectedResult controlExpectedResult)
+ throws Exception {
+ // test specifying with "id"
+ QueryResponse idRsp = getIdResponse(expectedResult);
+ QueryResponse idControlRsp = getIdResponse(controlExpectedResult);
+ assertExpected(expectedResult, idRsp, controlExpectedResult, idControlRsp);
+
+ // test specifying with "ids"
+ QueryResponse idsRsp = getIdsResponse(expectedResult);
+ QueryResponse idsControlRsp = getIdsResponse(controlExpectedResult);
+ assertExpected(expectedResult, idsRsp, controlExpectedResult, idsControlRsp);
+ }
+
+ @Test
+ public void testIdvsIds() throws Exception {
+ final String collection = "testIdvsIds";
+ final String collectionControl = collection + "Control";
+ setupCollectionWithDocSecurity(collection);
+ setupCollectionWithoutDocSecurity(collectionControl);
+ CloudSolrServer server = getCloudSolrServer(collection);
+ CloudSolrServer serverControl = getCloudSolrServer(collectionControl);
+
+ try {
+ for (CloudSolrServer s : new CloudSolrServer [] {server, serverControl}) {
+ DocLevelGenerator generator = new DocLevelGenerator(s.getDefaultCollection(), AUTH_FIELD);
+ generator.generateDocs(s, 100, "junit_role", "admin_role", 2);
+ }
+
+ // check that control collection does not filter
+ assertIdVsIds(new ExpectedResult(serverControl, new String[] {"2"}, 1),
+ new ExpectedResult(serverControl, new String[] {"2"}, 1));
+
+ // single id
+ assertIdVsIds(new ExpectedResult(server, new String[] {"1"}, 1),
+ new ExpectedResult(serverControl, new String[] {"1"}, 1));
+
+ // single id (invalid)
+ assertIdVsIds(new ExpectedResult(server, new String[] {"bogusId"}, 0),
+ new ExpectedResult(serverControl, new String[] {"bogusId"}, 0));
+
+ // single id (no permission)
+ assertIdVsIds(new ExpectedResult(server, new String[] {"2"}, 0),
+ new ExpectedResult(serverControl, new String[] {"2fake"}, 0));
+
+ // multiple ids (some invalid, some valid, some no permission)
+ assertIdVsIds(new ExpectedResult(server, new String[] {"bogus1", "1", "2"}, 1),
+ new ExpectedResult(serverControl, new String[] {"bogus1", "1", "bogus2"}, 1));
+ assertIdVsIds(new ExpectedResult(server, new String[] {"bogus1", "1", "2", "3"}, 2),
+ new ExpectedResult(serverControl, new String[] {"bogus1", "1", "bogus2", "3"}, 2));
+
+ // multiple ids (all invalid)
+ assertIdVsIds(new ExpectedResult(server, new String[] {"bogus1", "bogus2", "bogus3"}, 0),
+ new ExpectedResult(serverControl, new String[] {"bogus1", "bogus2", "bogus3"}, 0));
+
+ // multiple ids (all no permission)
+ assertIdVsIds(new ExpectedResult(server, new String[] {"2", "4", "6"}, 0),
+ new ExpectedResult(serverControl, new String[] {"bogus2", "bogus4", "bogus6"}, 0));
+
+ } finally {
+ server.shutdown();
+ serverControl.shutdown();
+ }
+ }
+
+ private void assertFlOnDocList(SolrDocumentList list, Set<String> expectedIds,
+ List<String> expectedFields) {
+ assertEquals("Doc list size should be: " + expectedIds.size(), expectedIds.size(), list.getNumFound());
+ for (SolrDocument doc : list) {
+ expectedIds.contains(doc.get("id"));
+ for (String field : expectedFields) {
+ assertNotNull("Field: " + field + " should not be null in doc: " + doc, doc.get(field));
+ }
+ assertEquals("doc should have: " + expectedFields.size() + " fields. Doc: " + doc,
+ expectedFields.size(), doc.getFieldNames().size());
+ }
+ }
+
+ private void assertFl(CloudSolrServer server, String [] ids, Set<String> expectedIds,
+ String fl, List<String> expectedFields) throws Exception {
+ {
+ QueryResponse idRsp = getIdResponse(new ExpectedResult(server, ids, expectedIds.size(), fl));
+ SolrDocumentList idList = idRsp.getResults();
+ assertFlOnDocList(idList, expectedIds, expectedFields);
+ }
+ {
+ QueryResponse idsRsp = getIdsResponse(new ExpectedResult(server, ids, expectedIds.size(), fl));
+ SolrDocumentList idsList = idsRsp.getResults();
+ assertFlOnDocList(idsList, expectedIds, expectedFields);
+ }
+ }
+
+ @Test
+ public void testFl() throws Exception {
+ final String collection = "testFl";
+ // FixMe: have to use one shard, because of a Solr bug where "fl" is not applied to
+ // multi-shard get requests
+ setupCollectionWithDocSecurity(collection, 1);
+ CloudSolrServer server = getCloudSolrServer(collection);
+
+ try {
+ DocLevelGenerator generator = new DocLevelGenerator(collection, AUTH_FIELD);
+ generator.generateDocs(server, 100, "junit_role", "admin_role", 2);
+ String [] ids = new String[] {"1", "3", "5"};
+
+ assertFl(server, ids, new HashSet<String>(Arrays.asList(ids)), "id", Arrays.asList("id"));
+ assertFl(server, ids, new HashSet<String>(Arrays.asList(ids)), null, Arrays.asList("id", "description", "_version_"));
+ // test transformer
+ assertFl(server, ids, new HashSet<String>(Arrays.asList(ids)), "id,mydescription:description", Arrays.asList("id", "mydescription"));
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ @Test
+ public void testNonCommitted() throws Exception {
+ final String collection = "testNonCommitted";
+ setupCollectionWithDocSecurity(collection, 1);
+ CloudSolrServer server = getCloudSolrServer(collection);
+
+ try {
+ DocLevelGenerator generator = new DocLevelGenerator(collection, AUTH_FIELD);
+ generator.generateDocs(server, 100, "junit_role", "admin_role", 2);
+
+ // make some uncommitted modifications and ensure they are reflected
+ server.deleteById("1");
+
+ SolrInputDocument doc2 = new SolrInputDocument();
+ doc2.addField("id", "2");
+ doc2.addField("description", "description2");
+ doc2.addField(AUTH_FIELD, "admin_role");
+
+ SolrInputDocument doc3 = new SolrInputDocument();
+ doc3.addField("id", "3");
+ doc3.addField("description", "description3");
+ doc3.addField(AUTH_FIELD, "junit_role");
+
+ SolrInputDocument doc200 = new SolrInputDocument();
+ doc200.addField("id", "200");
+ doc200.addField("description", "description200");
+ doc200.addField(AUTH_FIELD, "admin_role");
+ server.add(Arrays.asList(new SolrInputDocument [] {doc2, doc3, doc200}));
+
+ assertFl(server, new String[] {"1", "2", "3", "4", "5", "200"},
+ new HashSet<String>(Arrays.asList("2", "5", "200")), "id", Arrays.asList("id"));
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ private void assertConcurrentOnDocList(SolrDocumentList list, String authField, String expectedAuthFieldValue) {
+ for (SolrDocument doc : list) {
+ Collection<Object> authFieldValues = doc.getFieldValues(authField);
+ assertNotNull(authField + " should not be null. Doc: " + doc, authFieldValues);
+
+ boolean foundAuthFieldValue = false;
+ for (Object obj : authFieldValues) {
+ if (obj.toString().equals(expectedAuthFieldValue)) {
+ foundAuthFieldValue = true;
+ break;
+ }
+ }
+ assertTrue("Did not find: " + expectedAuthFieldValue + " in doc: " + doc, foundAuthFieldValue);
+ }
+ }
+
+ private void assertConcurrent(CloudSolrServer server, String [] ids, String authField, String expectedAuthFieldValue)
+ throws Exception {
+ {
+ QueryResponse idRsp = getIdResponse(new ExpectedResult(server, ids, -1, null));
+ SolrDocumentList idList = idRsp.getResults();
+ assertConcurrentOnDocList(idList, authField, expectedAuthFieldValue);
+ }
+ {
+ QueryResponse idsRsp = getIdsResponse(new ExpectedResult(server, ids, -1, null));
+ SolrDocumentList idsList = idsRsp.getResults();
+ assertConcurrentOnDocList(idsList, authField, expectedAuthFieldValue);
+ }
+ }
+
+ @Test
+ public void testConcurrentChanges() throws Exception {
+ final String collection = "testConcurrentChanges";
+ // Ensure the auth field is stored so we can check a consistent doc is returned
+ final String authField = "sentry_auth_stored";
+ System.setProperty("sentry.auth.field", authField);
+ setupCollectionWithDocSecurity(collection, 1);
+ CloudSolrServer server = getCloudSolrServer(collection);
+ int numQueries = 5;
+
+ try {
+ DocLevelGenerator generator = new DocLevelGenerator(collection, authField);
+ generator.generateDocs(server, 100, "junit_role", "admin_role", 2);
+
+ List<AuthFieldModifyThread> threads = new LinkedList<AuthFieldModifyThread>();
+ int docsToModify = 10;
+ for (int i = 0; i < docsToModify; ++i) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", Integer.toString(i));
+ doc.addField("description", "description" + Integer.toString(i));
+ doc.addField(authField, "junit_role");
+ server.add(doc);
+
+ threads.add(new AuthFieldModifyThread(server, doc,
+ authField, "junit_role", "admin_role"));
+ }
+ server.commit();
+
+ for (AuthFieldModifyThread thread : threads) {
+ thread.start();
+ }
+
+ // query
+ String [] ids = new String[docsToModify];
+ for (int j = 0; j < ids.length; ++j) {
+ ids[ j ] = Integer.toString(j);
+ }
+ for (int k = 0; k < numQueries; ++k) {
+ assertConcurrent(server, ids, authField, "admin_role");
+ }
+
+ for (AuthFieldModifyThread thread : threads) {
+ thread.setFinished();
+ thread.join();
+ }
+ } finally {
+ System.clearProperty("sentry.auth.field");
+ server.shutdown();
+ }
+ }
+
+ @Test
+ public void testSuperUser() throws Exception {
+ final String collection = "testSuperUser";
+ setupCollectionWithDocSecurity(collection, 1);
+ CloudSolrServer server = getCloudSolrServer(collection);
+ int docCount = 100;
+
+ try {
+ DocLevelGenerator generator = new DocLevelGenerator(collection, AUTH_FIELD);
+ generator.generateDocs(server, docCount, "junit_role", "admin_role", 2);
+
+ setAuthenticationUser("solr");
+ String [] ids = new String[docCount];
+ for (int i = 0; i < docCount; ++i) {
+ ids[ i ] = Integer.toString(i);
+ }
+ QueryResponse response = getIdResponse(new ExpectedResult(server, ids, docCount));
+ assertEquals("Wrong number of documents", docCount, response.getResults().getNumFound());
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ private class AuthFieldModifyThread extends Thread {
+ private CloudSolrServer server;
+ private SolrInputDocument doc;
+ private String authField;
+ private String authFieldValue0;
+ private String authFieldValue1;
+ private volatile boolean finished = false;
+
+ private AuthFieldModifyThread(CloudSolrServer server,
+ SolrInputDocument doc, String authField,
+ String authFieldValue0, String authFieldValue1) {
+ this.server = server;
+ this.doc = doc;
+ this.authField = authField;
+ this.authFieldValue0 = authFieldValue0;
+ this.authFieldValue1 = authFieldValue1;
+ }
+
+ @Override
+ public void run() {
+ while (!finished) {
+ if (rand.nextBoolean()) {
+ doc.setField(authField, authFieldValue0);
+ } else {
+ doc.setField(authField, authFieldValue1);
+ }
+ try {
+ server.add(doc);
+ } catch (SolrServerException sse) {
+ throw new RuntimeException(sse);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+
+ public void setFinished() {
+ finished = true;
+ }
+ }
+
+ private static class ExpectedResult {
+ public final CloudSolrServer server;
+ public final String [] ids;
+ public final int expectedDocs;
+ public final String fl;
+
+ public ExpectedResult(CloudSolrServer server, String [] ids, int expectedDocs) {
+ this(server, ids, expectedDocs, null);
+ }
+
+ public ExpectedResult(CloudSolrServer server, String [] ids, int expectedDocs, String fl) {
+ this.server = server;
+ this.ids = ids;
+ this.expectedDocs = expectedDocs;
+ this.fl = fl;
+ }
+ }
+}
diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
index 66449ff..c8bc32f 100644
--- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
+++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
@@ -216,6 +216,7 @@
<dynamicField name="*_c" type="currency" indexed="true" stored="true"/>
<dynamicField name="*_auth" type="string" indexed="true" stored="false" multiValued="true"/>
+ <dynamicField name="*_auth_stored" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
<dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/>
diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml
index 4459c0d..f07d494 100644
--- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml
+++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml
@@ -387,14 +387,14 @@
'soft' commit which only ensures that changes are visible
but does not ensure that data is synced to disk. This is
faster and more near-realtime friendly than a hard commit.
- -->
- <autoSoftCommit>
- <maxTime>${solr.autoSoftCommit.maxTime:1000}</maxTime>
+ -->
+ <autoSoftCommit>
+ <maxTime>${solr.autoSoftCommit.maxTime:20000}</maxTime>
</autoSoftCommit>
-
+
<!-- Update Related Event Listeners
-
+
Various IndexWriter related events can trigger Listeners to
take actions.
@@ -899,7 +899,7 @@
<!-- realtime get handler, guaranteed to return the latest stored fields of
any document, without the need to commit or open a new searcher. The
current implementation relies on the updateLog feature being enabled. -->
- <requestHandler name="/get" class="solr.RealTimeGetHandler">
+ <requestHandler name="/get" class="solr.SecureRealTimeGetHandler">
<lst name="defaults">
<str name="omitHeader">true</str>
<str name="wt">json</str>
@@ -1351,14 +1351,17 @@
<bool name="enabled">true</bool>
<!-- Field where the auth tokens are stored in the document -->
- <str name="sentryAuthField">sentry_auth</str>
+ <str name="sentryAuthField">${sentry.auth.field:sentry_auth}</str>
<!-- Auth token defined to allow any role to access the document.
Uncomment to enable. -->
<str name="allRolesToken">OR</str>
</searchComponent>
- <!-- A request handler for demonstrating the spellcheck component.
+ <searchComponent name="secureGet" class="org.apache.solr.handler.component.SecureRealTimeGetComponent" >
+ </searchComponent>
+
+ <!-- A request handler for demonstrating the spellcheck component.
NOTE: This is purely as an example. The whole purpose of the
SpellCheckComponent is to hook it into the request handler that
diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini b/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini
index bccc63e..a376cb8 100644
--- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini
+++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini
@@ -31,7 +31,7 @@
[roles]
junit_role = collection=admin, collection=collection1, collection=docLevelCollection, collection=allRolesCollection, collection=testUpdateDeleteOperations
docLevel_role = collection=docLevelCollection, collection=testUpdateDeleteOperations
-admin_role = collection=admin, collection=collection1, collection=sentryCollection, collection=sentryCollection_underlying1, collection=sentryCollection_underlying2, collection=docLevelCollection, collection=allRolesCollection, collection=testInvariantCollection, collection=testUpdateDeleteOperations, collection=testIndexlevelDoclevelOperations, collection=testUpdateDistribPhase
+admin_role = collection=admin, collection=collection1, collection=sentryCollection, collection=sentryCollection_underlying1, collection=sentryCollection_underlying2, collection=docLevelCollection, collection=allRolesCollection, collection=testInvariantCollection, collection=testUpdateDeleteOperations, collection=testIndexlevelDoclevelOperations, collection=testUpdateDistribPhase, collection=testIdvsIds, collection=testIdvsIdsControl, collection=testFl, collection=testNonCommitted, collection=testConcurrentChanges, collection=testSuperUser
sentryCollection_query_role = collection=sentryCollection->action=query
sentryCollection_update_role = collection=sentryCollection->action=update
sentryCollection_query_update_role = collection=sentryCollection->action=query, collection=sentryCollection->action=update
