UNOMI-522, UNOMI-628, UNOMI-629: event collector and aliases cleanup (#471)
* UNOMI-522: event collector endpoint is now handling current visitor session and profile exactly like context.json endpoint
* UNOMI-628, UNOMI-629: Cleanup profile alias mechanism, provide better migration and itest on the migrated data
* oops
* UNOMI-628, UNOMI-629: provide additional tests on profile aliases
diff --git a/api/src/main/java/org/apache/unomi/api/EventsCollectorRequest.java b/api/src/main/java/org/apache/unomi/api/EventsCollectorRequest.java
index 9eb4ee9..74f922e 100644
--- a/api/src/main/java/org/apache/unomi/api/EventsCollectorRequest.java
+++ b/api/src/main/java/org/apache/unomi/api/EventsCollectorRequest.java
@@ -34,6 +34,9 @@
@Pattern(regexp = ValidationPattern.TEXT_VALID_CHARACTERS_PATTERN)
private String sessionId;
+ @Pattern(regexp = ValidationPattern.TEXT_VALID_CHARACTERS_PATTERN)
+ private String profileId;
+
/**
* Retrieves the events to be processed.
*
@@ -67,4 +70,22 @@
this.sessionId = sessionId;
}
+ /**
+ * Retrieve the profileId passed along with the request. All events will be processed with this profileId as a
+ * default
+ *
+ * @return the identifier for the profile
+ */
+ public String getProfileId() {
+ return profileId;
+ }
+
+ /**
+ * Sets the profileId in the request.
+ *
+ * @param profileId an unique identifier for the profile
+ */
+ public void setProfileId(String profileId) {
+ this.profileId = profileId;
+ }
}
diff --git a/api/src/main/java/org/apache/unomi/api/Profile.java b/api/src/main/java/org/apache/unomi/api/Profile.java
index 7115bd5..ea95838 100644
--- a/api/src/main/java/org/apache/unomi/api/Profile.java
+++ b/api/src/main/java/org/apache/unomi/api/Profile.java
@@ -55,6 +55,10 @@
private Map<String, Integer> scores;
+ /**
+ * @deprecated since 2.0.0 merge mechanism is now based on profile aliases, and this property is not used anymore
+ */
+ @Deprecated
private String mergedWith;
private Map<String, Consent> consents = new LinkedHashMap<>();
@@ -201,18 +205,17 @@
}
/**
- * Retrieves the identifier of the profile this profile is merged with if any.
- *
- * @return the identifier of the profile this profile is merged with if any, {@code null} otherwise
+ * @deprecated since 2.0.0 merge mechanism is now based on profile aliases, and this property is not used anymore
*/
+ @Deprecated
public String getMergedWith() {
return mergedWith;
}
/**
- * TODO: should be removed from the API
- * @param mergedWith new value for mergedWith
+ * @deprecated since 2.0.0 merge mechanism is now based on profile aliases, and this property is not used anymore
*/
+ @Deprecated
public void setMergedWith(String mergedWith) {
this.mergedWith = mergedWith;
}
@@ -290,7 +293,6 @@
sb.append(", systemProperties=").append(systemProperties);
sb.append(", segments=").append(segments);
sb.append(", scores=").append(scores);
- sb.append(", mergedWith='").append(mergedWith).append('\'');
sb.append(", consents=").append(consents);
sb.append(", itemId='").append(itemId).append('\'');
sb.append(", itemType='").append(itemType).append('\'');
diff --git a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
index c3aa72a..a4af565 100644
--- a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
+++ b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
@@ -20,8 +20,9 @@
<details>${project.description}</details>
<feature prerequisite="true" dependency="false">wrap</feature>
<feature dependency="true">unomi-kar</feature>
- <bundle start-level="85">mvn:org.codehaus.groovy/groovy/3.0.3</bundle>
- <bundle start-level="85">mvn:org.codehaus.groovy/groovy-xml/3.0.3</bundle>
+ <bundle start-level="85">mvn:org.codehaus.groovy/groovy/${groovy.version}</bundle>
+ <bundle start-level="85">mvn:org.codehaus.groovy/groovy-xml/${groovy.version}</bundle>
+ <bundle start-level="85">mvn:org.codehaus.groovy/groovy-json/${groovy.version}</bundle>
<bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-services/${project.version}</bundle>
<bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-rest/${project.version}</bundle>
<bundle start-level="85">wrap:mvn:io.github.http-builder-ng/http-builder-ng-core/1.0.4</bundle>
diff --git a/extensions/groovy-actions/services/pom.xml b/extensions/groovy-actions/services/pom.xml
index 770d4da..cf747d6 100644
--- a/extensions/groovy-actions/services/pom.xml
+++ b/extensions/groovy-actions/services/pom.xml
@@ -86,6 +86,12 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-json</artifactId>
+ <version>${groovy.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
diff --git a/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/fetchers/profile/ProfileDataFetcher.java b/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/fetchers/profile/ProfileDataFetcher.java
index 0000065..6a8bf0b 100644
--- a/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/fetchers/profile/ProfileDataFetcher.java
+++ b/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/fetchers/profile/ProfileDataFetcher.java
@@ -53,8 +53,6 @@
profile = profileService.save(profile);
- profileService.addAliasToProfile(profile.getItemId(), profile.getItemId(), profileIDInput.getClient().getId());
-
return new CDPProfile(profile);
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
index 8e38424..c85a83a 100644
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
@@ -16,13 +16,13 @@
*/
package org.apache.unomi.itests;
-import org.apache.unomi.api.PartialList;
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.ProfileAlias;
+import org.apache.unomi.api.*;
import org.apache.unomi.api.query.Query;
import org.apache.unomi.api.services.DefinitionsService;
import org.apache.unomi.api.services.ProfileService;
import org.apache.unomi.persistence.spi.PersistenceService;
+import org.apache.unomi.schema.api.JsonSchemaWrapper;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -63,6 +63,11 @@
TestUtils.removeAllProfiles(definitionsService, persistenceService);
}
+ @After
+ public void tearDown() throws InterruptedException {
+ removeItems(Profile.class, ProfileAlias.class);
+ }
+
@Test
public void testProfileDelete() throws Exception {
Profile profile = new Profile();
@@ -219,4 +224,46 @@
}
}
+ @Test
+ public void testAliasCannotBeCreatedOnSameProfile() throws Exception {
+ String profileID = UUID.randomUUID().toString();
+ Profile profile = new Profile();
+ profile.setItemId(profileID);
+ profileService.save(profile);
+
+ keepTrying("Profile " + profileID + " not found in the required time", () -> profileService.load(profileID), Objects::nonNull,
+ DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ try {
+ profileService.addAliasToProfile(profileID, profileID, "defaultClientId");
+ fail("It should not be possible to create an Alias on the same profile ID");
+ } catch (Exception e) {
+ // do nothing, it's expected
+ }
+ }
+
+ @Test
+ public void testAliasCannotBeCreatedInCaseAlreadyExists() throws Exception {
+ String profileID = UUID.randomUUID().toString();
+ String alias = UUID.randomUUID().toString();
+ Profile profile = new Profile();
+ profile.setItemId(profileID);
+ profileService.save(profile);
+
+ keepTrying("Profile " + profileID + " not found in the required time", () -> profileService.load(profileID), Objects::nonNull,
+ DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ profileService.addAliasToProfile(profileID, alias, "defaultClientId");
+
+ keepTrying("Profile " + profileID + " not found in the required time", () -> profileService.load(alias), Objects::nonNull,
+ DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ String otherProfileId = UUID.randomUUID().toString();
+ try {
+ profileService.addAliasToProfile(otherProfileId, alias, "defaultClientId");
+ fail("It should not be possible to create an Alias when an alias already exists with same ID");
+ } catch (Exception e) {
+ // do nothing, it's expected
+ }
+ }
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
index faf6a87..921ae4a 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
@@ -28,6 +28,7 @@
import org.junit.Test;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -62,6 +63,7 @@
@After
public void cleanup() throws InterruptedException {
removeItems(Profile.class);
+ removeItems(ProfileAlias.class);
removeItems(Session.class);
removeItems(Event.class);
removeItems(Scope.class);
@@ -69,6 +71,7 @@
@Test
public void checkMigratedData() throws Exception {
+ checkMergedProfilesAliases();
checkProfileInterests();
checkScopeHaveBeenCreated();
checkFormEventRestructured();
@@ -82,6 +85,7 @@
*/
private void checkForMappingUpdates() throws IOException {
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scope/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-profilealias/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-segment/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scoring/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-campaign/_mapping", null).contains("\"entryCondition\":{\"type\":\"object\",\"enabled\":false}"));
@@ -215,4 +219,27 @@
}
}
}
+
+ /**
+ * Data set contains a master profile: 468ca2bf-7d24-41ea-9ef4-5b96f78207e4
+ * And two profiles that have been merged with this master profile: c33dec90-ffc9-4484-9e61-e42c323f268f and ac5b6b0f-afce-4c4f-9391-4ff0b891b254
+ */
+ private void checkMergedProfilesAliases() {
+ // Check that both profiles aliases have been created and the merged profiles are now deleted.
+ List<String> mergedProfiles = Arrays.asList("c33dec90-ffc9-4484-9e61-e42c323f268f", "ac5b6b0f-afce-4c4f-9391-4ff0b891b254");
+ String masterProfile = "468ca2bf-7d24-41ea-9ef4-5b96f78207e4";
+ for (String mergedProfile : mergedProfiles) {
+ // control the created alias
+ ProfileAlias alias = persistenceService.load(mergedProfile, ProfileAlias.class);
+ Assert.assertNotNull(alias);
+ Assert.assertEquals(alias.getProfileID(), masterProfile);
+
+ // control the merged profile do not exist anymore
+ Assert.assertNull(persistenceService.load(mergedProfile, Profile.class));
+ }
+
+ // Check master profile still exists a no alias have been created for him
+ Assert.assertNotNull(persistenceService.load(masterProfile, Profile.class));
+ Assert.assertNull(persistenceService.load(masterProfile, ProfileAlias.class));
+ }
}
diff --git a/itests/src/test/resources/migration/snapshots_repository.zip b/itests/src/test/resources/migration/snapshots_repository.zip
index e0a81a7..58eb87e 100644
--- a/itests/src/test/resources/migration/snapshots_repository.zip
+++ b/itests/src/test/resources/migration/snapshots_repository.zip
Binary files differ
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
index 2e21d8b..7520685 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
@@ -31,10 +31,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@@ -46,13 +42,9 @@
private EventService eventService;
private DefinitionsService definitionsService;
private PrivacyService privacyService;
- private ConfigSharingService configSharingService;
private int maxProfilesInOneMerge = -1;
public int execute(Action action, Event event) {
-// String profileIdCookieName = (String) configSharingService.getProperty("profileIdCookieName");
-// String profileIdCookieDomain = (String) configSharingService.getProperty("profileIdCookieDomain");
-// Integer profileIdCookieMaxAgeInSeconds = (Integer) configSharingService.getProperty("profileIdCookieMaxAgeInSeconds");
Profile profile = event.getProfile();
if (profile instanceof Persona || profile.isAnonymousProfile()) {
@@ -84,15 +76,7 @@
propertyCondition.setParameter("propertyName", "systemProperties." + mergeProfilePropertyName);
propertyCondition.setParameter("propertyValue", mergeProfilePropertyValue);
- Condition excludeMergedProfilesCondition = new Condition(definitionsService.getConditionType("profilePropertyCondition"));
- excludeMergedProfilesCondition.setParameter("comparisonOperator", "missing");
- excludeMergedProfilesCondition.setParameter("propertyName", "mergedWith");
-
- Condition c = new Condition(definitionsService.getConditionType("booleanCondition"));
- c.setParameter("operator", "and");
- c.setParameter("subConditions", Arrays.asList(propertyCondition, excludeMergedProfilesCondition));
-
- final List<Profile> profiles = persistenceService.query(c, "properties.firstVisit", Profile.class, 0, maxProfilesInOneMerge).getList();
+ final List<Profile> profiles = persistenceService.query(propertyCondition, "properties.firstVisit", Profile.class, 0, maxProfilesInOneMerge).getList();
// Check if the user switched to another profile
if (StringUtils.isNotEmpty(mergeProfilePreviousPropertyValue) && !mergeProfilePreviousPropertyValue.equals(mergeProfilePropertyValue)) {
@@ -111,14 +95,6 @@
}
logger.info("Different users, switch to " + profile.getItemId());
-
-// HttpServletResponse httpServletResponse = (HttpServletResponse) event.getAttributes().get(Event.HTTP_RESPONSE_ATTRIBUTE);
-// HttpServletRequest httpServletRequest = (HttpServletRequest) event.getAttributes().get(Event.HTTP_REQUEST_ATTRIBUTE);
-// if (httpServletRequest != null) {
-// sendProfileCookie(profile, httpServletResponse, profileIdCookieName, profileIdCookieDomain,
-// profileIdCookieMaxAgeInSeconds, profileIdCookieHttpOnly, httpServletRequest.isSecure());
-// }
-
// At the end of the merge, we must set the merged profile as profile event to process other Actions
event.setProfileId(profile.getItemId());
event.setProfile(profile);
@@ -157,13 +133,6 @@
// Profile has changed
if (forceEventProfileAsMaster || !masterProfile.getItemId().equals(profileId)) {
- HttpServletResponse httpServletResponse = (HttpServletResponse) event.getAttributes().get(Event.HTTP_RESPONSE_ATTRIBUTE);
- // we still send back the current profile cookie. It will be changed on the next request to the ContextServlet.
- // The current profile will be deleted only then because we cannot delete it right now (too soon)
-// if (httpServletRequest != null) {
-// sendProfileCookie(profile, httpServletResponse, profileIdCookieName, profileIdCookieDomain,
-// profileIdCookieMaxAgeInSeconds, profileIdCookieHttpOnly, httpServletRequest.isSecure());
-// }
final String masterProfileId = masterProfile.getItemId();
// At the end of the merge, we must set the merged profile as profile event to process other Actions
@@ -242,20 +211,6 @@
}
}
-// private static void sendProfileCookie(Profile profile, ServletResponse response, String profileIdCookieName, String profileIdCookieDomain, int cookieAgeInSeconds) {
-// if (response != null && response instanceof HttpServletResponse) {
-// HttpServletResponse httpServletResponse = (HttpServletResponse) response;
-// if (!(profile instanceof Persona)) {
-// httpServletResponse.addHeader("Set-Cookie",
-// profileIdCookieName + "=" + profile.getItemId() +
-// "; Path=/" +
-// "; Max-Age=" + cookieAgeInSeconds +
-// (StringUtils.isNotBlank(profileIdCookieDomain) ? ("; Domain=" + profileIdCookieDomain) : "") +
-// "; SameSite=Lax");
-// }
-// }
-// }
-
public void setProfileService(ProfileService profileService) {
this.profileService = profileService;
}
@@ -276,12 +231,7 @@
this.definitionsService = definitionsService;
}
- public void setConfigSharingService(ConfigSharingService configSharingService) {
- this.configSharingService = configSharingService;
- }
-
public void setMaxProfilesInOneMerge(String maxProfilesInOneMerge) {
this.maxProfilesInOneMerge = Integer.parseInt(maxProfilesInOneMerge);
}
-
}
diff --git a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index fd9512a..3222eb6 100644
--- a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -315,7 +315,6 @@
<property name="persistenceService" ref="persistenceService"/>
<property name="definitionsService" ref="definitionsService"/>
<property name="privacyService" ref="privacyService"/>
- <property name="configSharingService" ref="configSharingService"/>
<property name="maxProfilesInOneMerge" value="${base.maxProfilesInOneMerge}"/>
</bean>
</service>
diff --git a/rest/src/main/java/org/apache/unomi/rest/endpoints/ContextJsonEndpoint.java b/rest/src/main/java/org/apache/unomi/rest/endpoints/ContextJsonEndpoint.java
index da3debb..6207845 100644
--- a/rest/src/main/java/org/apache/unomi/rest/endpoints/ContextJsonEndpoint.java
+++ b/rest/src/main/java/org/apache/unomi/rest/endpoints/ContextJsonEndpoint.java
@@ -20,7 +20,6 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.StringUtils;
import org.apache.cxf.rs.security.cors.CrossOriginResourceSharing;
import org.apache.unomi.api.*;
import org.apache.unomi.api.conditions.Condition;
@@ -29,8 +28,7 @@
import org.apache.unomi.rest.exception.InvalidRequestException;
import org.apache.unomi.rest.service.RestServiceUtils;
import org.apache.unomi.schema.api.SchemaService;
-import org.apache.unomi.utils.Changes;
-import org.apache.unomi.utils.HttpUtils;
+import org.apache.unomi.utils.EventsRequestContext;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
@@ -38,8 +36,6 @@
import javax.jws.WebService;
import javax.servlet.ServletContext;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*;
@@ -53,7 +49,6 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
@WebService
@Consumes(MediaType.APPLICATION_JSON)
@@ -63,9 +58,7 @@
public class ContextJsonEndpoint {
private static final Logger logger = LoggerFactory.getLogger(ContextJsonEndpoint.class.getName());
- private static final String DEFAULT_CLIENT_ID = "defaultClientId";
-
- private boolean sanitizeConditions = Boolean
+ private final boolean sanitizeConditions = Boolean
.parseBoolean(System.getProperty("org.apache.unomi.security.personalization.sanitizeConditions", "true"));
@Context
@@ -76,18 +69,12 @@
HttpServletResponse response;
@Reference
- private ProfileService profileService;
- @Reference
private PrivacyService privacyService;
@Reference
- private EventService eventService;
- @Reference
private RulesService rulesService;
@Reference
private PersonalizationService personalizationService;
@Reference
- private ConfigSharingService configSharingService;
- @Reference
private RestServiceUtils restServiceUtils;
@Reference
private SchemaService schemaService;
@@ -147,10 +134,11 @@
@Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
@Path("/context.json")
public ContextResponse contextJSONAsPost(ContextRequest contextRequest,
- @QueryParam("personaId") String personaId,
- @QueryParam("sessionId") String sessionId,
- @QueryParam("timestamp") Long timestampAsLong, @QueryParam("invalidateProfile") boolean invalidateProfile,
- @QueryParam("invalidateSession") boolean invalidateSession) {
+ @QueryParam("personaId") String personaId,
+ @QueryParam("sessionId") String sessionId,
+ @QueryParam("timestamp") Long timestampAsLong,
+ @QueryParam("invalidateProfile") boolean invalidateProfile,
+ @QueryParam("invalidateSession") boolean invalidateSession) {
// Schema validation
ObjectNode paramsAsJson = JsonNodeFactory.instance.objectNode();
@@ -159,233 +147,57 @@
if (!schemaService.isValid(paramsAsJson.toString(), "https://unomi.apache.org/schemas/json/rest/requestIds/1-0-0")) {
throw new InvalidRequestException("Invalid parameter", "Invalid received data");
}
+
+ // Generate timestamp
Date timestamp = new Date();
if (timestampAsLong != null) {
timestamp = new Date(timestampAsLong);
}
- // Handle persona
- Profile profile = null;
- Session session = null;
+ // init ids
String profileId = null;
- if (personaId != null) {
- PersonaWithSessions personaWithSessions = profileService.loadPersonaWithSessions(personaId);
- if (personaWithSessions == null) {
- logger.error("Couldn't find persona, please check your personaId parameter");
- profile = null;
- } else {
- profile = personaWithSessions.getPersona();
- session = personaWithSessions.getLastSession();
- }
- }
-
String scope = null;
if (contextRequest != null) {
- if (contextRequest.getSource() != null) {
- scope = contextRequest.getSource().getScope();
- }
-
- if (contextRequest.getSessionId() != null) {
- sessionId = contextRequest.getSessionId();
- }
-
+ scope = contextRequest.getSource() != null ? contextRequest.getSource().getScope() : scope;
+ sessionId = contextRequest.getSessionId() != null ? contextRequest.getSessionId() : sessionId;
profileId = contextRequest.getProfileId();
}
- if (profileId == null) {
- // Get profile id from the cookie
- profileId = restServiceUtils.getProfileIdCookieValue(request);
- }
- if (profileId == null && sessionId == null && personaId == null) {
- logger.error(
- "Couldn't find profileId, sessionId or personaId in incoming request! Stopped processing request. See debug level for more information");
- if (logger.isDebugEnabled()) {
- logger.debug("Request dump: {}", HttpUtils.dumpRequestInfo(request));
- }
- throw new BadRequestException("Couldn't find profileId, sessionId or personaId in incoming request!");
- }
+ // build public context, profile + session creation/anonymous etc ...
+ EventsRequestContext eventsRequestContext = restServiceUtils.initEventsRequest(scope, sessionId, profileId,
+ personaId, invalidateProfile, invalidateSession, request, response, timestamp);
- int changes = EventService.NO_CHANGE;
-
- // Not a persona, resolve profile now
- boolean profileCreated = false;
-
- if (profile == null) {
- if (profileId == null || invalidateProfile) {
- // no profileId cookie was found or the profile has to be invalidated, we generate a new one and create the profile in the profile service
- profile = createNewProfile(null, timestamp);
- profileCreated = true;
- } else {
- profile = profileService.load(profileId);
- if (profile == null) {
- // this can happen if we have an old cookie but have reset the server,
- // or if we merged the profiles and somehow this cookie didn't get updated.
- profile = createNewProfile(profileId, timestamp);
- profileCreated = true;
- } else {
- Changes changesObject = checkMergedProfile(profile, session);
- changes |= changesObject.getChangeType();
- profile = changesObject.getProfile();
- }
- }
-
- Profile sessionProfile;
- if (StringUtils.isNotBlank(sessionId) && !invalidateSession) {
- session = profileService.loadSession(sessionId, timestamp);
- if (session != null) {
- sessionProfile = session.getProfile();
-
- boolean anonymousSessionProfile = sessionProfile.isAnonymousProfile();
- if (!profile.isAnonymousProfile() && !anonymousSessionProfile && !profile.getItemId()
- .equals(sessionProfile.getItemId())) {
- // Session user has been switched, profile id in cookie is not up to date
- // We must reload the profile with the session ID as some properties could be missing from the session profile
- // #personalIdentifier
- profile = profileService.load(sessionProfile.getItemId());
- }
-
- // Handle anonymous situation
- Boolean requireAnonymousBrowsing = privacyService.isRequireAnonymousBrowsing(profile);
- if (requireAnonymousBrowsing && anonymousSessionProfile) {
- // User wants to browse anonymously, anonymous profile is already set.
- } else if (requireAnonymousBrowsing && !anonymousSessionProfile) {
- // User wants to browse anonymously, update the sessionProfile to anonymous profile
- sessionProfile = privacyService.getAnonymousProfile(profile);
- session.setProfile(sessionProfile);
- changes |= EventService.SESSION_UPDATED;
- } else if (!requireAnonymousBrowsing && anonymousSessionProfile) {
- // User does not want to browse anonymously anymore, update the sessionProfile to real profile
- sessionProfile = profile;
- session.setProfile(sessionProfile);
- changes |= EventService.SESSION_UPDATED;
- } else if (!requireAnonymousBrowsing && !anonymousSessionProfile) {
- // User does not want to browse anonymously, use the real profile. Check that session contains the current profile.
- sessionProfile = profile;
- if (!session.getProfileId().equals(sessionProfile.getItemId())) {
- changes |= EventService.SESSION_UPDATED;
- }
- session.setProfile(sessionProfile);
- }
- }
- }
-
- if (session == null || invalidateSession) {
- sessionProfile = privacyService.isRequireAnonymousBrowsing(profile) ? privacyService.getAnonymousProfile(profile) : profile;
-
- if (StringUtils.isNotBlank(sessionId)) {
- // Only save session and send event if a session id was provided, otherwise keep transient session
- session = new Session(sessionId, sessionProfile, timestamp, scope);
- changes |= EventService.SESSION_UPDATED;
- Event event = new Event("sessionCreated", session, profile, scope, null, session, null, timestamp, false);
- if (sessionProfile.isAnonymousProfile()) {
- // Do not keep track of profile in event
- event.setProfileId(null);
- }
- event.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request);
- event.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response);
- if (logger.isDebugEnabled()) {
- logger.debug("Received event {} for profile={} session={} target={} timestamp={}", event.getEventType(),
- profile.getItemId(), session.getItemId(), event.getTarget(), timestamp);
- }
- changes |= eventService.send(event);
- }
- }
-
- if (profileCreated) {
- changes |= EventService.PROFILE_UPDATED;
-
- Event profileUpdated = new Event("profileUpdated", session, profile, scope, null, profile, timestamp);
- profileUpdated.setPersistent(false);
- profileUpdated.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request);
- profileUpdated.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response);
- profileUpdated.getAttributes().put(Event.CLIENT_ID_ATTRIBUTE, DEFAULT_CLIENT_ID);
-
- if (logger.isDebugEnabled()) {
- logger.debug("Received event {} for profile={} {} target={} timestamp={}", profileUpdated.getEventType(),
- profile.getItemId(), " session=" + (session != null ? session.getItemId() : null), profileUpdated.getTarget(),
- timestamp);
- }
- changes |= eventService.send(profileUpdated);
- }
- }
-
+ // Build response
ContextResponse contextResponse = new ContextResponse();
- contextResponse.setProfileId(profile.getItemId());
- if (session != null) {
- contextResponse.setSessionId(session.getItemId());
+ if (contextRequest != null) {
+ eventsRequestContext = processContextRequest(contextRequest, contextResponse, eventsRequestContext);
+ }
+
+ // finalize request, save profile and session if necessary and return profileId cookie in response
+ restServiceUtils.finalizeEventsRequest(eventsRequestContext, false);
+
+ contextResponse.setProfileId(eventsRequestContext.getProfile().getItemId());
+ if (eventsRequestContext.getSession() != null) {
+ contextResponse.setSessionId(eventsRequestContext.getSession().getItemId());
} else if (sessionId != null) {
contextResponse.setSessionId(sessionId);
}
-
- if (contextRequest != null) {
- Changes changesObject = handleRequest(contextRequest, session, profile, contextResponse, request, response, timestamp);
- changes |= changesObject.getChangeType();
- profile = changesObject.getProfile();
- }
-
- if ((changes & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) {
- profileService.save(profile);
- contextResponse.setProfileId(profile.getItemId());
-
- if (profileCreated) {
- String clientId = contextRequest != null && contextRequest.getClientId() != null ? contextRequest.getClientId() : DEFAULT_CLIENT_ID;
- String profileMasterId = profile.getMergedWith() != null ? profile.getMergedWith() : profile.getItemId();
- profileService.addAliasToProfile(profileMasterId, profile.getItemId(), clientId );
- }
- }
- if ((changes & EventService.SESSION_UPDATED) == EventService.SESSION_UPDATED && session != null) {
- profileService.saveSession(session);
- contextResponse.setSessionId(session.getItemId());
- }
-
- if ((changes & EventService.ERROR) == EventService.ERROR) {
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- }
- // Set profile cookie
- if (!(profile instanceof Persona)) {
- response.setHeader("Set-Cookie", HttpUtils.getProfileCookieString(profile, configSharingService, request.isSecure()));
- }
return contextResponse;
}
- private Changes checkMergedProfile(Profile profile, Session session) {
- int changes = EventService.NO_CHANGE;
- if (profile.getMergedWith() != null && !privacyService.isRequireAnonymousBrowsing(profile) && !profile.isAnonymousProfile()) {
- Profile currentProfile = profile;
- String masterProfileId = profile.getMergedWith();
- Profile masterProfile = profileService.load(masterProfileId);
- if (masterProfile != null) {
- logger.info("Current profile {} was merged with profile {}, replacing profile in session", currentProfile.getItemId(),
- masterProfileId);
- profile = masterProfile;
- if (session != null) {
- session.setProfile(profile);
- changes = EventService.SESSION_UPDATED;
- }
- } else {
- logger.warn("Couldn't find merged profile {}, falling back to profile {}", masterProfileId, currentProfile.getItemId());
- profile.setMergedWith(null);
- changes = EventService.PROFILE_UPDATED;
- }
- }
+ private EventsRequestContext processContextRequest(ContextRequest contextRequest, ContextResponse data, EventsRequestContext eventsRequestContext) {
- return new Changes(changes, profile);
- }
+ processOverrides(contextRequest, eventsRequestContext.getProfile(), eventsRequestContext.getSession());
- private Changes handleRequest(ContextRequest contextRequest, Session session, Profile profile, ContextResponse data,
- ServletRequest request, ServletResponse response, Date timestamp) {
-
- processOverrides(contextRequest, profile, session);
-
- Changes changes = restServiceUtils.handleEvents(contextRequest.getEvents(), session, profile, request, response, timestamp);
- data.setProcessedEvents(changes.getProcessedItems());
+ eventsRequestContext = restServiceUtils.performEventsRequest(contextRequest.getEvents(), eventsRequestContext);
+ data.setProcessedEvents(eventsRequestContext.getProcessedItems());
List<PersonalizationService.PersonalizedContent> filterNodes = contextRequest.getFilters();
if (filterNodes != null) {
data.setFilteringResults(new HashMap<>());
for (PersonalizationService.PersonalizedContent personalizedContent : sanitizePersonalizedContentObjects(filterNodes)) {
data.getFilteringResults()
- .put(personalizedContent.getId(), personalizationService.filter(profile, session, personalizedContent));
+ .put(personalizedContent.getId(), personalizationService.filter(eventsRequestContext.getProfile(), eventsRequestContext.getSession(), personalizedContent));
}
}
@@ -393,34 +205,31 @@
if (personalizations != null) {
data.setPersonalizations(new HashMap<>());
for (PersonalizationService.PersonalizationRequest personalization : sanitizePersonalizations(personalizations)) {
- PersonalizationResult personalizationResult = personalizationService.personalizeList(profile, session, personalization);
- changes.setChangeType(changes.getChangeType() | personalizationResult.getChangeType());
- data.getPersonalizations()
- .put(personalization.getId(), personalizationResult.getContentIds());
+ PersonalizationResult personalizationResult = personalizationService.personalizeList(eventsRequestContext.getProfile(), eventsRequestContext.getSession(), personalization);
+ eventsRequestContext.addChanges(personalizationResult.getChangeType());
+ data.getPersonalizations().put(personalization.getId(), personalizationResult.getContentIds());
}
}
- profile = changes.getProfile();
-
if (contextRequest.isRequireSegments()) {
- data.setProfileSegments(profile.getSegments());
+ data.setProfileSegments(eventsRequestContext.getProfile().getSegments());
}
if (contextRequest.isRequireScores()) {
- data.setProfileScores(profile.getScores());
+ data.setProfileScores(eventsRequestContext.getProfile().getScores());
}
if (contextRequest.getRequiredProfileProperties() != null) {
- Map<String, Object> profileProperties = new HashMap<>(profile.getProperties());
+ Map<String, Object> profileProperties = new HashMap<>(eventsRequestContext.getProfile().getProperties());
if (!contextRequest.getRequiredProfileProperties().contains("*")) {
profileProperties.keySet().retainAll(contextRequest.getRequiredProfileProperties());
}
data.setProfileProperties(profileProperties);
}
- if (session != null) {
- data.setSessionId(session.getItemId());
+ if (eventsRequestContext.getSession() != null) {
+ data.setSessionId(eventsRequestContext.getSession().getItemId());
if (contextRequest.getRequiredSessionProperties() != null) {
- Map<String, Object> sessionProperties = new HashMap<>(session.getProperties());
+ Map<String, Object> sessionProperties = new HashMap<>(eventsRequestContext.getSession().getProperties());
if (!contextRequest.getRequiredSessionProperties().contains("*")) {
sessionProperties.keySet().retainAll(contextRequest.getRequiredSessionProperties());
}
@@ -428,16 +237,16 @@
}
}
- if (!(profile instanceof Persona)) {
+ if (!(eventsRequestContext.getProfile() instanceof Persona)) {
data.setTrackedConditions(rulesService.getTrackedConditions(contextRequest.getSource()));
} else {
data.setTrackedConditions(Collections.emptySet());
}
- data.setAnonymousBrowsing(privacyService.isRequireAnonymousBrowsing(profile));
- data.setConsents(profile.getConsents());
+ data.setAnonymousBrowsing(privacyService.isRequireAnonymousBrowsing(eventsRequestContext.getProfile()));
+ data.setConsents(eventsRequestContext.getProfile().getConsents());
- return changes;
+ return eventsRequestContext;
}
/**
@@ -467,17 +276,6 @@
}
}
- private Profile createNewProfile(String existingProfileId, Date timestamp) {
- Profile profile;
- String profileId = existingProfileId;
- if (profileId == null) {
- profileId = UUID.randomUUID().toString();
- }
- profile = new Profile(profileId);
- profile.setProperty("firstVisit", timestamp);
- return profile;
- }
-
public void destroy() {
logger.info("Context servlet shutdown.");
}
diff --git a/rest/src/main/java/org/apache/unomi/rest/endpoints/EventsCollectorEndpoint.java b/rest/src/main/java/org/apache/unomi/rest/endpoints/EventsCollectorEndpoint.java
index 8911402..f727b35 100644
--- a/rest/src/main/java/org/apache/unomi/rest/endpoints/EventsCollectorEndpoint.java
+++ b/rest/src/main/java/org/apache/unomi/rest/endpoints/EventsCollectorEndpoint.java
@@ -21,28 +21,16 @@
import org.apache.cxf.rs.security.cors.CrossOriginResourceSharing;
import org.apache.unomi.api.Event;
import org.apache.unomi.api.EventsCollectorRequest;
-import org.apache.unomi.api.Persona;
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.Session;
-import org.apache.unomi.api.services.ConfigSharingService;
-import org.apache.unomi.api.services.EventService;
-import org.apache.unomi.api.services.PrivacyService;
-import org.apache.unomi.api.services.ProfileService;
import org.apache.unomi.rest.exception.InvalidRequestException;
import org.apache.unomi.rest.models.EventCollectorResponse;
import org.apache.unomi.rest.service.RestServiceUtils;
-import org.apache.unomi.utils.Changes;
+import org.apache.unomi.utils.EventsRequestContext;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import javax.jws.WebService;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import javax.validation.Valid;
-import javax.validation.constraints.NotNull;
-import javax.ws.rs.BadRequestException;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.OPTIONS;
@@ -54,7 +42,6 @@
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.Date;
-import java.util.UUID;
@WebService
@Produces(MediaType.APPLICATION_JSON + ";charset=UTF-8")
@@ -63,17 +50,8 @@
@Path("/")
@Component(service = EventsCollectorEndpoint.class, property = "osgi.jaxrs.resource=true")
public class EventsCollectorEndpoint {
- private static final Logger logger = LoggerFactory.getLogger(EventsCollectorEndpoint.class.getName());
@Reference
- private EventService eventService;
- @Reference
- private ProfileService profileService;
- @Reference
- private PrivacyService privacyService;
- @Reference
- private ConfigSharingService configSharingService;
- @Reference
private RestServiceUtils restServiceUtils;
@Context
@@ -114,87 +92,34 @@
if (sessionId == null) {
sessionId = request.getParameter("sessionId");
}
- Session session = null;
- if (sessionId != null) {
- session = profileService.loadSession(sessionId, timestamp);
- }
- Profile profile = null;
- if (session == null) {
- String scope = "systemscope";
- // Get the first available scope that is not equal to systemscope to create the session otherwise systemscope will be used
- for (Event event : eventsCollectorRequest.getEvents()) {
- if (StringUtils.isNotBlank(event.getEventType())) {
- if (StringUtils.isNotBlank(event.getScope()) && !event.getScope().equals("systemscope")) {
- scope = event.getScope();
- break;
- } else if (event.getSource() != null && StringUtils.isNotBlank(event.getSource().getScope()) && !event.getSource()
- .getScope().equals("systemscope")) {
- scope = event.getSource().getScope();
- break;
- }
- }
- }
- logger.debug("scope is now {}", scope);
- String cookieProfileId = restServiceUtils.getProfileIdCookieValue(request);
- if (StringUtils.isNotBlank(cookieProfileId)) {
- profile = profileService.load(cookieProfileId);
- }
- if (profile == null) {
- // Create non persisted profile to create the session
- profile = new Profile("temp_" + UUID.randomUUID().toString());
- profile.setProperty("firstVisit", timestamp);
- }
- /*
- // Create anonymous profile so we don't keep track of the temp profile anywhere
- Profile anonymousProfile = privacyService.getAnonymousProfile(profile);
- // Create new session which should not be persisted as well as the temp profile
- session = new Session(sessionId, anonymousProfile, timestamp, scope);
- if (logger.isDebugEnabled()) {
- logger.debug("No session found for sessionId={}, creating new session!", sessionId);
- }
- */
- } else {
- Profile sessionProfile = session.getProfile();
- final String errorMessage = String
- .format("No valid profile found or persona found for profileId=%s, aborting request !", session.getProfileId());
- if (sessionProfile.getItemId() != null) {
- // Reload up-to-date profile
- profile = profileService.load(sessionProfile.getItemId());
- if (profile == null || profile instanceof Persona) {
- logger.error(errorMessage);
- throw new BadRequestException(errorMessage);
- }
- } else {
- // Session uses anonymous profile, try to find profile from cookie
- String cookieProfileId = restServiceUtils.getProfileIdCookieValue(request);
- if (StringUtils.isNotBlank(cookieProfileId)) {
- profile = profileService.load(cookieProfileId);
- }
- if (profile == null) {
- logger.error(errorMessage);
- throw new BadRequestException(errorMessage);
+ String profileId = eventsCollectorRequest.getProfileId();
+ // Get the first available scope that is not equal to systemscope otherwise systemscope will be used
+ String scope = "systemscope";
+ for (Event event : eventsCollectorRequest.getEvents()) {
+ if (StringUtils.isNotBlank(event.getEventType())) {
+ if (StringUtils.isNotBlank(event.getScope()) && !event.getScope().equals("systemscope")) {
+ scope = event.getScope();
+ break;
+ } else if (event.getSource() != null &&
+ StringUtils.isNotBlank(event.getSource().getScope()) &&
+ !event.getSource().getScope().equals("systemscope")) {
+ scope = event.getSource().getScope();
+ break;
}
}
}
- Changes changesObject = restServiceUtils
- .handleEvents(eventsCollectorRequest.getEvents(), session, profile, request, response, timestamp);
- int changes = changesObject.getChangeType();
- profile = changesObject.getProfile();
+ // build public context, profile + session creation/anonymous etc ...
+ EventsRequestContext eventsRequestContext = restServiceUtils.initEventsRequest(scope, sessionId, profileId,
+ null, false, false, request, response, timestamp);
- if ((changes & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) {
- profileService.save(profile);
- }
- if ((changes & EventService.SESSION_UPDATED) == EventService.SESSION_UPDATED && session != null) {
- profileService.saveSession(session);
- }
- if ((changes & EventService.ERROR) == EventService.ERROR) {
- String errorMessage = "Error processing events. Total number of processed events: " + changesObject.getProcessedItems() + "/"
- + eventsCollectorRequest.getEvents().size();
- throw new BadRequestException(errorMessage);
- }
+ // process events
+ eventsRequestContext = restServiceUtils.performEventsRequest(eventsCollectorRequest.getEvents(), eventsRequestContext);
- return new EventCollectorResponse(changes);
+ // finalize request
+ restServiceUtils.finalizeEventsRequest(eventsRequestContext, true);
+
+ return new EventCollectorResponse(eventsRequestContext.getChanges());
}
}
diff --git a/rest/src/main/java/org/apache/unomi/rest/service/RestServiceUtils.java b/rest/src/main/java/org/apache/unomi/rest/service/RestServiceUtils.java
index e64db14..0e6a9ba 100644
--- a/rest/src/main/java/org/apache/unomi/rest/service/RestServiceUtils.java
+++ b/rest/src/main/java/org/apache/unomi/rest/service/RestServiceUtils.java
@@ -18,19 +18,65 @@
package org.apache.unomi.rest.service;
import org.apache.unomi.api.Event;
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.Session;
-import org.apache.unomi.utils.Changes;
+import org.apache.unomi.utils.EventsRequestContext;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
import java.util.Date;
import java.util.List;
+/**
+ * Utility service for Public REST endpoints
+ */
public interface RestServiceUtils {
- String getProfileIdCookieValue(HttpServletRequest httpServletRequest);
+ /**
+ * This method is used to initialize the context for a request that would require executing events.
+ *
+ * It will load existing profile/session for current user or build new ones if necessary
+ * IT will also handle anonymous profile/session preferences in case there is specific ones.
+ * It will also handle persona in case it is provided
+ * And finally it will provide a contextual bean named: EventsRequestContext,
+ * that will contain all the required information for the next steps of the request, like: processing the events
+ *
+ * @param scope the current scope (mandatory, in case session need to be created)
+ * @param sessionId the current sessionId (mandatory)
+ * @param profileId the current profileId (optional in case profile doesn't exists yet for incoming visitor)
+ * @param personaId the current personaId (optional in case we don't want to apply persona on current request)
+ * @param invalidateProfile true in case we want to invalidate the current visitor profile, false otherwise
+ * @param invalidateSession true in case we want to invalidate the current visitor session, false otherwise
+ * @param request the current request
+ * @param response the current request response
+ * @param timestamp the current date, for timestamp the current visitor data
+ *
+ * @return the built EventsRequestContext
+ */
+ EventsRequestContext initEventsRequest(String scope, String sessionId, String profileId, String personaId,
+ boolean invalidateProfile, boolean invalidateSession,
+ HttpServletRequest request, HttpServletResponse response,
+ Date timestamp);
- Changes handleEvents(List<Event> events, Session session, Profile profile, ServletRequest request, ServletResponse response,
- Date timestamp);
+ /**
+ * Execute the list of events using the dedicated eventsRequestContext
+ * @param events the list of events to he executed
+ * @param eventsRequestContext the current EventsRequestContext
+ * @return an updated version of the current eventsRequestContext
+ */
+ EventsRequestContext performEventsRequest(List<Event> events, EventsRequestContext eventsRequestContext);
+
+ /**
+ * At the end of an events requests we want to save/update the profile and/or the session depending on the changes
+ * Also we want to return a cookie about current visitor profile ID
+ *
+ * @param eventsRequestContext the current EventsRequestContext
+ * @param crashOnError true if we want to throw an Exception in case of errors during events execution,
+ * false otherwise (otherwise, no exception, but just an error code directly returned to the HTTP response)
+ */
+ void finalizeEventsRequest(EventsRequestContext eventsRequestContext, boolean crashOnError);
+
+ /**
+ * Try to extract the current visitor profileId from the current request cookies.
+ * @param httpServletRequest the current HTTP request
+ * @return the profileId if found in the cookies, null otherwise
+ */
+ String getProfileIdCookieValue(HttpServletRequest httpServletRequest);
}
diff --git a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
index fae86da..75a98d1 100644
--- a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
+++ b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java
@@ -17,32 +17,35 @@
package org.apache.unomi.rest.service.impl;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import org.apache.unomi.api.Event;
-import org.apache.unomi.api.Persona;
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.Session;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.unomi.api.*;
import org.apache.unomi.api.services.ConfigSharingService;
import org.apache.unomi.api.services.EventService;
import org.apache.unomi.api.services.PrivacyService;
+import org.apache.unomi.api.services.ProfileService;
import org.apache.unomi.rest.exception.InvalidRequestException;
import org.apache.unomi.rest.service.RestServiceUtils;
import org.apache.unomi.schema.api.SchemaService;
-import org.apache.unomi.utils.Changes;
+import org.apache.unomi.utils.HttpUtils;
+import org.apache.unomi.utils.EventsRequestContext;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.BadRequestException;
import java.util.Date;
import java.util.List;
+import java.util.UUID;
@Component(service = RestServiceUtils.class)
public class RestServiceUtilsImpl implements RestServiceUtils {
+ private static final String DEFAULT_CLIENT_ID = "defaultClientId";
+
private static final Logger logger = LoggerFactory.getLogger(RestServiceUtilsImpl.class.getName());
@Reference
@@ -55,8 +58,12 @@
private EventService eventService;
@Reference
+ private ProfileService profileService;
+
+ @Reference
SchemaService schemaService;
+ @Override
public String getProfileIdCookieValue(HttpServletRequest httpServletRequest) {
String cookieProfileId = null;
@@ -78,59 +85,243 @@
}
@Override
- public Changes handleEvents(List<Event> events, Session session, Profile profile, ServletRequest request, ServletResponse response,
- Date timestamp) {
- List<String> filteredEventTypes = privacyService.getFilteredEventTypes(profile);
+ public EventsRequestContext initEventsRequest(String scope, String sessionId, String profileId, String personaId,
+ boolean invalidateProfile, boolean invalidateSession,
+ HttpServletRequest request, HttpServletResponse response, Date timestamp) {
- String thirdPartyId = eventService
- .authenticateThirdPartyServer(((HttpServletRequest) request).getHeader("X-Unomi-Peer"), request.getRemoteAddr());
+ // Build context
+ EventsRequestContext eventsRequestContext = new EventsRequestContext(timestamp, null, null, request, response);
- int changes = EventService.NO_CHANGE;
+ // Handle persona
+ if (personaId != null) {
+ PersonaWithSessions personaWithSessions = profileService.loadPersonaWithSessions(personaId);
+ if (personaWithSessions == null) {
+ logger.error("Couldn't find persona, please check your personaId parameter");
+ } else {
+ eventsRequestContext.setProfile(personaWithSessions.getPersona());
+ eventsRequestContext.setSession(personaWithSessions.getLastSession());
+ }
+ }
+
+ if (profileId == null) {
+ // Get profile id from the cookie
+ profileId = getProfileIdCookieValue(request);
+ }
+
+ if (profileId == null && sessionId == null && personaId == null) {
+ logger.error("Couldn't find profileId, sessionId or personaId in incoming request! Stopped processing request. See debug level for more information");
+ if (logger.isDebugEnabled()) {
+ logger.debug("Request dump: {}", HttpUtils.dumpRequestInfo(request));
+ }
+ throw new BadRequestException("Couldn't find profileId, sessionId or personaId in incoming request!");
+ }
+
+ boolean profileCreated = false;
+ if (eventsRequestContext.getProfile() == null) {
+ if (profileId == null || invalidateProfile) {
+ // no profileId cookie was found or the profile has to be invalidated, we generate a new one and create the profile in the profile service
+ eventsRequestContext.setProfile(createNewProfile(null, timestamp));
+ profileCreated = true;
+ } else {
+ eventsRequestContext.setProfile(profileService.load(profileId));
+ if (eventsRequestContext.getProfile() == null) {
+ // this can happen if we have an old cookie but have reset the server,
+ // or if we merged the profiles and somehow this cookie didn't get updated.
+ eventsRequestContext.setProfile(createNewProfile(profileId, timestamp));
+ profileCreated = true;
+ }
+ }
+
+ // Try to recover existing session
+ Profile sessionProfile;
+ if (StringUtils.isNotBlank(sessionId) && !invalidateSession) {
+
+ eventsRequestContext.setSession(profileService.loadSession(sessionId, timestamp));
+ if (eventsRequestContext.getSession() != null) {
+
+ sessionProfile = eventsRequestContext.getSession().getProfile();
+ boolean anonymousSessionProfile = sessionProfile.isAnonymousProfile();
+ if (!eventsRequestContext.getProfile().isAnonymousProfile() &&
+ !anonymousSessionProfile &&
+ !eventsRequestContext.getProfile().getItemId().equals(sessionProfile.getItemId())) {
+ // Session user has been switched, profile id in cookie is not up to date
+ // We must reload the profile with the session ID as some properties could be missing from the session profile
+ // #personalIdentifier
+ eventsRequestContext.setProfile(profileService.load(sessionProfile.getItemId()));
+ }
+
+ // Handle anonymous situation
+ Boolean requireAnonymousBrowsing = privacyService.isRequireAnonymousBrowsing(eventsRequestContext.getProfile());
+ if (requireAnonymousBrowsing && anonymousSessionProfile) {
+ // User wants to browse anonymously, anonymous profile is already set.
+ } else if (requireAnonymousBrowsing && !anonymousSessionProfile) {
+ // User wants to browse anonymously, update the sessionProfile to anonymous profile
+ sessionProfile = privacyService.getAnonymousProfile(eventsRequestContext.getProfile());
+ eventsRequestContext.getSession().setProfile(sessionProfile);
+ eventsRequestContext.addChanges(EventService.SESSION_UPDATED);
+ } else if (!requireAnonymousBrowsing && anonymousSessionProfile) {
+ // User does not want to browse anonymously anymore, update the sessionProfile to real profile
+ sessionProfile = eventsRequestContext.getProfile();
+ eventsRequestContext.getSession().setProfile(sessionProfile);
+ eventsRequestContext.addChanges(EventService.SESSION_UPDATED);
+ } else if (!requireAnonymousBrowsing && !anonymousSessionProfile) {
+ // User does not want to browse anonymously, use the real profile. Check that session contains the current profile.
+ sessionProfile = eventsRequestContext.getProfile();
+ if (!eventsRequestContext.getSession().getProfileId().equals(sessionProfile.getItemId())) {
+ eventsRequestContext.addChanges(EventService.SESSION_UPDATED);
+ }
+ eventsRequestContext.getSession().setProfile(sessionProfile);
+ }
+ }
+ }
+
+ // Try to create new session
+ if (eventsRequestContext.getSession() == null || invalidateSession) {
+ sessionProfile = privacyService.isRequireAnonymousBrowsing(eventsRequestContext.getProfile()) ?
+ privacyService.getAnonymousProfile(eventsRequestContext.getProfile()) : eventsRequestContext.getProfile();
+
+ if (StringUtils.isNotBlank(sessionId)) {
+ // Only save session and send event if a session id was provided, otherwise keep transient session
+ eventsRequestContext.setSession(new Session(sessionId, sessionProfile, timestamp, scope));
+ eventsRequestContext.addChanges(EventService.SESSION_UPDATED);
+
+ Event event = new Event("sessionCreated", eventsRequestContext.getSession(), eventsRequestContext.getProfile(),
+ scope, null, eventsRequestContext.getSession(), null, timestamp, false);
+ if (sessionProfile.isAnonymousProfile()) {
+ // Do not keep track of profile in event
+ event.setProfileId(null);
+ }
+ event.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request);
+ event.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Received event {} for profile={} session={} target={} timestamp={}", event.getEventType(),
+ eventsRequestContext.getProfile().getItemId(), eventsRequestContext.getSession().getItemId(), event.getTarget(), timestamp);
+ }
+ eventsRequestContext.addChanges(eventService.send(event));
+ }
+ }
+
+ // Handle new profile creation
+ if (profileCreated) {
+ eventsRequestContext.addChanges(EventService.PROFILE_UPDATED);
+
+ Event profileUpdated = new Event("profileUpdated", eventsRequestContext.getSession(), eventsRequestContext.getProfile(),
+ scope, null, eventsRequestContext.getProfile(), timestamp);
+ profileUpdated.setPersistent(false);
+ profileUpdated.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request);
+ profileUpdated.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response);
+ profileUpdated.getAttributes().put(Event.CLIENT_ID_ATTRIBUTE, DEFAULT_CLIENT_ID);
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Received event {} for profile={} {} target={} timestamp={}", profileUpdated.getEventType(),
+ eventsRequestContext.getProfile().getItemId(),
+ " session=" + (eventsRequestContext.getSession() != null ? eventsRequestContext.getSession().getItemId() : null),
+ profileUpdated.getTarget(), timestamp);
+ }
+ eventsRequestContext.addChanges(eventService.send(profileUpdated));
+ }
+ }
+
+ return eventsRequestContext;
+ }
+
+ @Override
+ public EventsRequestContext performEventsRequest(List<Event> events, EventsRequestContext eventsRequestContext) {
+ List<String> filteredEventTypes = privacyService.getFilteredEventTypes(eventsRequestContext.getProfile());
+ String thirdPartyId = eventService.authenticateThirdPartyServer(eventsRequestContext.getRequest().getHeader("X-Unomi-Peer"),
+ eventsRequestContext.getRequest().getRemoteAddr());
+
// execute provided events if any
- int processedEventsCnt = 0;
- if (events != null && !(profile instanceof Persona)) {
+ if (events != null && !(eventsRequestContext.getProfile() instanceof Persona)) {
+ // set Total items on context
+ eventsRequestContext.setTotalItems(events.size());
+
for (Event event : events) {
- processedEventsCnt++;
+ eventsRequestContext.setProcessedItems(eventsRequestContext.getProcessedItems() + 1);
+
if (event.getEventType() != null) {
- Event eventToSend = new Event(event.getEventType(), session, profile, event.getScope(), event.getSource(),
- event.getTarget(), event.getProperties(), timestamp, event.isPersistent());
+ Event eventToSend = new Event(event.getEventType(), eventsRequestContext.getSession(), eventsRequestContext.getProfile(), event.getScope(), event.getSource(),
+ event.getTarget(), event.getProperties(), eventsRequestContext.getTimestamp(), event.isPersistent());
eventToSend.setFlattenedProperties(event.getFlattenedProperties());
if (!eventService.isEventAllowed(event, thirdPartyId)) {
logger.warn("Event is not allowed : {}", event.getEventType());
continue;
}
if (thirdPartyId != null && event.getItemId() != null) {
- eventToSend = new Event(event.getItemId(), event.getEventType(), session, profile, event.getScope(),
- event.getSource(), event.getTarget(), event.getProperties(), timestamp, event.isPersistent());
+ eventToSend = new Event(event.getItemId(), event.getEventType(), eventsRequestContext.getSession(), eventsRequestContext.getProfile(), event.getScope(),
+ event.getSource(), event.getTarget(), event.getProperties(), eventsRequestContext.getTimestamp(), event.isPersistent());
eventToSend.setFlattenedProperties(event.getFlattenedProperties());
}
if (filteredEventTypes != null && filteredEventTypes.contains(event.getEventType())) {
logger.debug("Profile is filtering event type {}", event.getEventType());
continue;
}
- if (profile.isAnonymousProfile()) {
+ if (eventsRequestContext.getProfile().isAnonymousProfile()) {
// Do not keep track of profile in event
eventToSend.setProfileId(null);
}
- eventToSend.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request);
- eventToSend.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response);
- logger.debug("Received event " + event.getEventType() + " for profile=" + profile.getItemId() + " session=" + (
- session != null ? session.getItemId() : null) + " target=" + event.getTarget() + " timestamp=" + timestamp);
- changes |= eventService.send(eventToSend);
+ eventToSend.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, eventsRequestContext.getRequest());
+ eventToSend.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, eventsRequestContext.getResponse());
+ logger.debug("Received event " + event.getEventType() + " for profile=" + eventsRequestContext.getProfile().getItemId() + " session=" + (
+ eventsRequestContext.getSession() != null ? eventsRequestContext.getSession().getItemId() : null) +
+ " target=" + event.getTarget() + " timestamp=" + eventsRequestContext.getTimestamp());
+ eventsRequestContext.addChanges(eventService.send(eventToSend));
// If the event execution changes the profile we need to update it so the next event use the right profile
- if ((changes & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) {
- profile = eventToSend.getProfile();
+ if ((eventsRequestContext.getChanges() & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) {
+ eventsRequestContext.setProfile(eventToSend.getProfile());
}
- if ((changes & EventService.ERROR) == EventService.ERROR) {
+ if ((eventsRequestContext.getChanges() & EventService.ERROR) == EventService.ERROR) {
//Don't count the event that failed
- processedEventsCnt--;
- logger.error("Error processing events. Total number of processed events: {}/{}", processedEventsCnt, events.size());
+ eventsRequestContext.setProcessedItems(eventsRequestContext.getProcessedItems() - 1);
+ logger.error("Error processing events. Total number of processed events: {}/{}", eventsRequestContext.getProcessedItems(), eventsRequestContext.getTotalItems());
break;
}
}
}
}
- return new Changes(changes, processedEventsCnt, profile);
+
+ return eventsRequestContext;
+ }
+
+ @Override
+ public void finalizeEventsRequest(EventsRequestContext eventsRequestContext, boolean crashOnError) {
+ // in case of changes on profile, persist the profile
+ if ((eventsRequestContext.getChanges() & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) {
+ profileService.save(eventsRequestContext.getProfile());
+ }
+
+ // in case of changes on session, persist the session
+ if ((eventsRequestContext.getChanges() & EventService.SESSION_UPDATED) == EventService.SESSION_UPDATED && eventsRequestContext.getSession() != null) {
+ profileService.saveSession(eventsRequestContext.getSession());
+ }
+
+ // In case of error, return an error message
+ if ((eventsRequestContext.getChanges() & EventService.ERROR) == EventService.ERROR) {
+ if (crashOnError) {
+ String errorMessage = "Error processing events. Total number of processed events: " + eventsRequestContext.getProcessedItems() + "/"
+ + eventsRequestContext.getTotalItems();
+ throw new BadRequestException(errorMessage);
+ } else {
+ eventsRequestContext.getResponse().setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ // Set profile cookie
+ if (!(eventsRequestContext.getProfile() instanceof Persona)) {
+ eventsRequestContext.getResponse().setHeader("Set-Cookie",
+ HttpUtils.getProfileCookieString(eventsRequestContext.getProfile(), configSharingService, eventsRequestContext.getRequest().isSecure()));
+ }
+ }
+
+ private Profile createNewProfile(String existingProfileId, Date timestamp) {
+ Profile profile;
+ String profileId = existingProfileId;
+ if (profileId == null) {
+ profileId = UUID.randomUUID().toString();
+ }
+ profile = new Profile(profileId);
+ profile.setProperty("firstVisit", timestamp);
+ return profile;
}
}
diff --git a/rest/src/main/java/org/apache/unomi/utils/Changes.java b/rest/src/main/java/org/apache/unomi/utils/Changes.java
deleted file mode 100644
index b333430..0000000
--- a/rest/src/main/java/org/apache/unomi/utils/Changes.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.unomi.utils;
-
-import org.apache.unomi.api.Profile;
-
-/**
- * This class is a simple object to get the updated profile without the need of reloading it
- *
- * @author dgaillard
- */
-public class Changes {
- private int changeType;
- private int processedItems;
- private Profile profile;
-
- public Changes(int changeType, Profile profile) {
- this(changeType,0,profile);
- }
-
- public Changes(int changeType, int processedItems, Profile profile) {
- this.changeType = changeType;
- this.processedItems = processedItems;
- this.profile = profile;
- }
-
- public int getChangeType() {
- return changeType;
- }
-
- public void setChangeType(int changeType) {
- this.changeType = changeType;
- }
-
- public int getProcessedItems() {
- return processedItems;
- }
-
- public Profile getProfile() {
- return profile;
- }
-}
diff --git a/rest/src/main/java/org/apache/unomi/utils/EventsRequestContext.java b/rest/src/main/java/org/apache/unomi/utils/EventsRequestContext.java
new file mode 100644
index 0000000..188f42d
--- /dev/null
+++ b/rest/src/main/java/org/apache/unomi/utils/EventsRequestContext.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.utils;
+
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.Session;
+import org.apache.unomi.api.services.EventService;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.util.Date;
+
+/**
+ * This is a bean that maintain the current situation during a request that contains events to be processed.
+ * It's in charge to hold an up to date Session + Profile for the current request, but also the status of the events executions:
+ * - changes
+ * - number of events processed
+ */
+public class EventsRequestContext {
+
+ private Date timestamp;
+ private Profile profile;
+ private Session session;
+ private HttpServletRequest request;
+ private HttpServletResponse response;
+ private int changes;
+ private int totalItems;
+ private int processedItems;
+
+ private EventsRequestContext() {
+ }
+
+ public EventsRequestContext(Date timestamp, Profile profile, Session session, HttpServletRequest request, HttpServletResponse response) {
+ this.timestamp = timestamp;
+ this.profile = profile;
+ this.session = session;
+ this.request = request;
+ this.response = response;
+ this.changes = EventService.NO_CHANGE;
+ this.totalItems = 0;
+ this.processedItems = 0;
+ }
+
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(Date timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public Profile getProfile() {
+ return profile;
+ }
+
+ public void setProfile(Profile profile) {
+ this.profile = profile;
+ }
+
+ public Session getSession() {
+ return session;
+ }
+
+ public void setSession(Session session) {
+ this.session = session;
+ }
+
+ public int getChanges() {
+ return changes;
+ }
+
+ public void addChanges(int changes) {
+ this.changes |= changes;
+ }
+
+ public int getTotalItems() {
+ return totalItems;
+ }
+
+ public void setTotalItems(int totalItems) {
+ this.totalItems = totalItems;
+ }
+
+ public int getProcessedItems() {
+ return processedItems;
+ }
+
+ public void setProcessedItems(int processedItems) {
+ this.processedItems = processedItems;
+ }
+
+ public HttpServletRequest getRequest() {
+ return request;
+ }
+
+ public void setRequest(HttpServletRequest request) {
+ this.request = request;
+ }
+
+ public HttpServletResponse getResponse() {
+ return response;
+ }
+
+ public void setResponse(HttpServletResponse response) {
+ this.response = response;
+ }
+}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
index cdfa8c8..a11189a 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
@@ -563,7 +563,14 @@
@Override
public void addAliasToProfile(String profileID, String alias, String clientID) {
+ if (Objects.equals(alias, profileID)) {
+ throw new IllegalArgumentException("Alias cannot be created on itself, please use an alias different from the profile ID");
+ }
+
ProfileAlias profileAlias = persistenceService.load(alias, ProfileAlias.class);
+ if (profileAlias != null && !Objects.equals(profileAlias.getProfileID(), profileID)) {
+ throw new IllegalArgumentException("Alias \"" + alias + "\" already used by profile with ID = \"" + profileAlias.getProfileID() + "\"");
+ }
if (profileAlias == null) {
profileAlias = new ProfileAlias();
@@ -578,8 +585,6 @@
profileAlias.setModifiedTime(creationTime);
persistenceService.save(profileAlias);
- } else if (!Objects.equals(profileAlias.getProfileID(), profileID)) {
- throw new IllegalArgumentException("Alias \"" + alias + "\" already used by profile with ID = \"" + profileID + "\"");
}
}
@@ -678,12 +683,6 @@
if (persona) {
persistenceService.remove(profileId, Persona.class);
} else {
- Condition mergeCondition = new Condition(definitionsService.getConditionType("profilePropertyCondition"));
- mergeCondition.setParameter("propertyName", "mergedWith");
- mergeCondition.setParameter("comparisonOperator", "equals");
- mergeCondition.setParameter("propertyValue", profileId);
- persistenceService.removeByQuery(mergeCondition, Profile.class);
-
Condition removeAliasesCondition = new Condition(definitionsService.getConditionType("profileAliasesPropertyCondition"));
removeAliasesCondition.setParameter("propertyName", "profileID");
removeAliasesCondition.setParameter("comparisonOperator", "equals");
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index e3e863d..1b73ab2 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -86,6 +86,13 @@
<version>${groovy.version}</version>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-json</artifactId>
+ <version>${groovy.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
index da32fab..41f37aa 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
@@ -53,7 +53,7 @@
m.put(CONFIG_ES_ADDRESS, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: http://localhost:9200): ", "http://localhost:9200"));
m.put(CONFIG_ES_LOGIN, new MigrationConfigProperty("Enter ElasticSearch TARGET login (default: none): ", ""));
m.put(CONFIG_ES_PASSWORD, new MigrationConfigProperty("Enter ElasticSearch TARGET password (default: none): ", ""));
- m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ?", null));
+ m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ? (yes/no)", null));
m.put(INDEX_PREFIX, new MigrationConfigProperty("Enter ElasticSearch Unomi indices prefix (default: context): ", "context"));
m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 3): ", "3"));
m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index 585bd20..ea5afdf 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -54,7 +54,6 @@
@Service
public class Migrate implements Action {
-
@Reference
Session session;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
deleted file mode 100644
index b1de181..0000000
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.shell.migration.impl;
-
-import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
-import org.apache.unomi.shell.migration.utils.MigrationUtils;
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.osgi.framework.BundleContext;
-
-import java.io.IOException;
-import java.net.URI;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-public class MigrationTo200 implements Migration {
-
- private CloseableHttpClient httpClient;
- private Session session;
- private String esAddress;
- private String indexPrefix;
- private BundleContext bundleContext;
- private MigrationConfig migrationConfig;
-
- @Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- this.httpClient = httpClient;
- this.session = session;
- this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- this.indexPrefix = migrationConfig.getString(MigrationConfig.INDEX_PREFIX, session);
- this.bundleContext = bundleContext;
- this.migrationConfig = migrationConfig;
-
- doExecute();
- }
-
- private void doExecute() throws IOException {
- Set<String> indexes = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, indexPrefix + "-event-");
- createScopeMapping();
- createScopes(getSetOfScopes(indexes));
- createProfileAliasDocumentsFromProfile();
- }
-
- private boolean scopeIndexNotExists() throws IOException {
- final HttpGet httpGet = new HttpGet(esAddress + "/" + indexPrefix + "-scope");
-
- httpGet.addHeader("Accept", "application/json");
- httpGet.addHeader("Content-Type", "application/json");
-
- try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
- return response.getStatusLine().getStatusCode() != HttpStatus.SC_OK;
- }
- }
-
- private void createScopeMapping() throws IOException {
-
- if (scopeIndexNotExists()) {
- System.out.println("Creation for index = \"" + indexPrefix + "-scope\" starting.");
- final HttpPut httpPost = new HttpPut(esAddress + "/" + indexPrefix + "-scope");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/json");
-
- String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json");
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json");
- // We intentionally extract setting from profile index, because the scope index doesnt exist yet, and all indices share the same configuration regarding shards, replicas, etc ..
- String request = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, indexPrefix + "-profile", mapping);
-
- httpPost.setEntity(new StringEntity(request));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println(indexPrefix + "-scope has been correctly created");
- } else {
- System.out.println(
- "Failed to create the index " + indexPrefix + "-scope.Code:" + response.getStatusLine().getStatusCode());
- throw new RuntimeException("Can not create the scope index. Stop the execution of the migration.");
- }
- }
- } else {
- System.out.println("The scope index already exists. Skipping the creation of this index");
- }
-
- }
-
- private void createScopes(Set<String> scopes) throws IOException {
- final StringBuilder body = new StringBuilder();
- String saveScopeBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/bulkSaveScope.ndjson");
- scopes.forEach(scope -> body.append(saveScopeBody.replace("$scope", scope)));
-
- final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix + "-scope/_bulk");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/x-ndjson");
-
- httpPost.setEntity(new StringEntity(body.toString()));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope successfully finished");
- } else {
- System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope has failed" + response.getStatusLine()
- .getStatusCode());
- }
- }
- }
-
- private Set<String> getSetOfScopes(Set<String> indices) throws IOException {
- String joinedIndices = String.join(",", indices);
- final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices + "/_search");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/json");
-
- String request = MigrationUtils.resourceAsString(bundleContext,"requestBody/searchScope.json");
-
- httpPost.setEntity(new StringEntity(request));
-
- Set<String> scopes = new HashSet<>();
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Getting the \"scope\" values from the events successfully finished. " + "Number of scope to create: "
- + responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
- scopes = StreamSupport
- .stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
- false).map(bucketElement -> ((JSONObject) bucketElement).getString("key")).collect(Collectors.toSet());
- } else {
- System.out.println(
- "Getting the \"scope\" values from the event has failed. Code: " + response.getStatusLine().getStatusCode());
- }
- }
- return scopes;
- }
-
- private void createProfileAliasDocumentsFromProfile() throws IOException {
- System.out.println("Migration \"Create profileAlias from profile\" started");
- Instant migrationTime = Instant.now();
- int size = 1000;
- doProcessProfiles(migrationTime, size);
- System.out.println("Migration \"Create profileAlias from profile\" completed.");
- }
-
- private void doProcessProfiles(Instant migrationTime, int offset) throws IOException {
- CloseableHttpResponse response = null;
- try {
- response = httpClient.execute(createSearchRequest(offset));
-
- while (true) {
- JSONObject responseAsJson = getResponseAsJSON(response);
-
- String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id"): null;
- JSONArray hits = getProfileHits(responseAsJson);
-
- if (hits.length() == 0) {
- if (scrollId != null) {
- CloseableHttpResponse deleteScrollResponse = httpClient.execute(createDeleteScrollRequest(scrollId));
- if (deleteScrollResponse != null) {
- deleteScrollResponse.close();
- }
- }
- break;
- }
-
- StringBuilder bulkCreateRequest = new StringBuilder();
- for (Object o : hits) {
- JSONObject hit = (JSONObject) o;
- if (hit.has("_source")) {
- JSONObject profile = hit.getJSONObject("_source");
- if (profile.has("itemId")) {
- String itemId = profile.getString("itemId");
- String bulkSaveProfileAliases = MigrationUtils.resourceAsString(bundleContext,"requestBody/bulkSaveProfileAliases.ndjson");
- bulkCreateRequest.append(bulkSaveProfileAliases.
- replace("$itemId", itemId).
- replace("$migrationTime", migrationTime.toString()));
- }
- }
- }
-
- CloseableHttpResponse bulkResponse = httpClient.execute(createProfileAliasRequest(bulkCreateRequest.toString()));
- if (bulkResponse != null) {
- bulkResponse.close();
- }
-
- response = httpClient.execute(createSearchRequestWithScrollId(scrollId));
- }
- } finally {
- if (response != null) {
- response.close();
- }
- }
- }
-
- private JSONObject getResponseAsJSON(CloseableHttpResponse response) throws IOException {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- return new JSONObject(EntityUtils.toString(response.getEntity()));
- }
- return new JSONObject();
- }
-
- private JSONArray getProfileHits(JSONObject responseAsJson) {
- if (responseAsJson.has("hits")) {
- JSONObject hitsObject = responseAsJson.getJSONObject("hits");
- if (hitsObject.has("hits")) {
- return hitsObject.getJSONArray("hits");
- }
- }
- return new JSONArray();
- }
-
- private HttpPost createSearchRequestWithScrollId(final String scrollId) throws IOException {
- final String requestBody = "{\n" +
- " \"scroll_id\": \"" + scrollId + "\",\n" +
- " \"scroll\": \"1h\"\n" +
- "}";
-
- final HttpPost request = new HttpPost(esAddress + "/_search/scroll");
-
- request.addHeader("Accept", "application/json");
- request.addHeader("Content-Type", "application/json");
- request.setEntity(new StringEntity(requestBody));
-
- return request;
- }
-
- private HttpGet createSearchRequest(int size) {
- return new HttpGet(esAddress + "/context-profile/_search?&scroll=1h&_source_includes=itemId&size=" + size);
- }
-
- private HttpEntityEnclosingRequestBase createDeleteScrollRequest(final String scrollId) throws IOException {
- final HttpEntityEnclosingRequestBase deleteRequest = new HttpEntityEnclosingRequestBase() {
- @Override
- public String getMethod() {
- return "DELETE";
- }
- };
-
- deleteRequest.setURI(URI.create(esAddress + "/_search/scroll"));
- deleteRequest.setEntity(new StringEntity("{ \"scroll_id\": \"" + scrollId + "\" }"));
- deleteRequest.addHeader("Accept", "application/json");
- deleteRequest.addHeader("Content-Type", "application/json");
-
- return deleteRequest;
- }
-
- private HttpPost createProfileAliasRequest(String bulkRequestAsString) throws IOException {
- final HttpPost bulkRequest = new HttpPost(esAddress + "/context-profilealias/_bulk");
-
- bulkRequest.addHeader("Accept", "application/json");
- bulkRequest.addHeader("Content-Type", "application/json");
- bulkRequest.setEntity(new StringEntity(bulkRequestAsString));
-
- return bulkRequest;
- }
-}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index e93998b..6297a89 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -23,6 +23,7 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
import org.json.JSONObject;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
@@ -87,6 +88,13 @@
}
}
+ public static boolean indexExists(CloseableHttpClient httpClient, String esAddress, String indexName) throws IOException {
+ final HttpGet httpGet = new HttpGet(esAddress + "/" + indexName);
+ try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
+ return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
+ }
+ }
+
public static Set<String> getIndexesPrefixedBy(CloseableHttpClient httpClient, String esAddress, String prefix) throws IOException {
try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
@@ -156,6 +164,45 @@
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
}
+ public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
+ String response = HttpUtils.executePostRequest(httpClient, esAddress + queryURL + "?scroll=" + scrollDuration, query, null);
+
+ while (true) {
+ JSONObject responseAsJson = new JSONObject(response);
+ String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id"): null;
+ JSONArray hits = new JSONArray();
+ if (responseAsJson.has("hits")) {
+ JSONObject hitsObject = responseAsJson.getJSONObject("hits");
+ if (hitsObject.has("hits")) {
+ hits = hitsObject.getJSONArray("hits");
+ }
+ }
+
+ // no more results, delete scroll
+ if (hits.length() == 0) {
+ if (scrollId != null) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/_search/scroll/" + scrollId, null);
+ }
+ break;
+ }
+
+ // execute callback
+ if (scrollCallback != null) {
+ scrollCallback.execute(hits.toString());
+ }
+
+ // scroll
+ response = HttpUtils.executePostRequest(httpClient, esAddress + "/_search/scroll", "{\n" +
+ " \"scroll_id\": \"" + scrollId + "\",\n" +
+ " \"scroll\": \"" + scrollDuration + "\"\n" +
+ "}", null);
+ }
+ }
+
+ public interface ScrollCallback {
+ void execute(String hits);
+ }
+
private static String getScriptPart(String painlessScript) {
return ", \"script\": {\"source\": \"" + painlessScript + "\", \"lang\": \"painless\"}";
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-00-scopesAndProfileAliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-00-scopesAndProfileAliases.groovy
deleted file mode 100644
index 80d66d9..0000000
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-00-scopesAndProfileAliases.groovy
+++ /dev/null
@@ -1,20 +0,0 @@
-import org.apache.unomi.shell.migration.impl.MigrationTo200
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-new MigrationTo200().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
new file mode 100644
index 0000000..6ad9e9b
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -0,0 +1,70 @@
+import groovy.json.JsonSlurper
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+import java.time.Instant
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Instant migrationTime = Instant.now();
+def jsonSlurper = new JsonSlurper()
+String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String aliasIndex = indexPrefix + "-profilealias"
+String profileIndex = indexPrefix + "-profile"
+
+// create alias index
+if (!MigrationUtils.indexExists(httpClient, esAddress, aliasIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, profileIndex, mapping)
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + aliasIndex, newIndexSettings, null)
+
+ // scroll search on profiles merged
+ String profileMergedSearchRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_search.json")
+ MigrationUtils.scrollQuery(httpClient, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", new MigrationUtils.ScrollCallback() {
+ @Override
+ void execute(String hits) {
+ // create aliases for those merged profiles and delete them.
+ def jsonHits = jsonSlurper.parseText(hits)
+ ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " profile alias to create")
+ final StringBuilder bulkSaveRequest = new StringBuilder()
+ jsonHits.each {
+ jsonHit -> {
+ // check that master still exists before creating alias:
+ def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + profileIndex + "/_doc/" + jsonHit._source.mergedWith, null))
+ if (masterProfile.found) {
+ bulkSaveRequest.append(aliasSaveBulkRequest
+ .replace("##itemId##", jsonHit._source.itemId)
+ .replace("##profileId##", jsonHit._source.mergedWith)
+ .replace("##migrationTime##", migrationTime.toString()))
+ }
+ }
+ }
+ if (bulkSaveRequest.length() > 0) {
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ }
+ }
+ })
+
+ // delete existing merged profiles
+ String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
new file mode 100644
index 0000000..771c3ec
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -0,0 +1,50 @@
+import groovy.json.JsonSlurper
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def jsonSlurper = new JsonSlurper()
+String searchScopesRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/scope_search.json")
+String saveScopeRequestBulk = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scope_save_bulk.ndjson")
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String scopeIndex = indexPrefix + "-scope"
+
+// Create scope index:
+if (!MigrationUtils.indexExists(httpClient, esAddress, scopeIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, indexPrefix + "-profile", mapping)
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + scopeIndex, newIndexSettings, null)
+
+ // search existing scopes from event
+ def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
+ ConsoleUtils.printMessage(session, "Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
+
+ // create scopes
+ def buckets = searchResponse.aggregations.scopes.buckets
+ if (buckets != null && buckets.size() > 0) {
+ final StringBuilder bulkSaveRequest = new StringBuilder()
+ buckets.each {
+ bucket -> bulkSaveRequest.append(saveScopeRequestBulk.replace("##scope##", bucket.key))
+ }
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-globalReindex.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/alias_save_bulk.ndjson b/tools/shell-commands/src/main/resources/requestBody/2.0.0/alias_save_bulk.ndjson
new file mode 100644
index 0000000..f69e1b9
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/alias_save_bulk.ndjson
@@ -0,0 +1,2 @@
+{ "create" : { "_id": "##itemId##" }}
+{ "itemId": "##itemId##", "itemType": "profileAlias", "profileID": "##profileId##", "scope": null, "clientID": "defaultClientId", "creationTime": "##migrationTime##", "modifiedTime": "##migrationTime##"}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_delete.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_delete.json
new file mode 100644
index 0000000..a6d7cb7
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_delete.json
@@ -0,0 +1,7 @@
+{
+ "query": {
+ "exists": {
+ "field": "mergedWith"
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_search.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_search.json
new file mode 100644
index 0000000..d6598b3
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_search.json
@@ -0,0 +1,8 @@
+{
+ "size": 1000,
+ "query": {
+ "exists": {
+ "field": "mergedWith"
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_save_bulk.ndjson b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_save_bulk.ndjson
new file mode 100644
index 0000000..9785285
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_save_bulk.ndjson
@@ -0,0 +1,2 @@
+{"create": {"_id": "##scope##"}}
+{"itemId": "##scope##", "itemType": "scope", "metadata": {"id": "##scope##" }}
diff --git a/tools/shell-commands/src/main/resources/requestBody/searchScope.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_search.json
similarity index 100%
rename from tools/shell-commands/src/main/resources/requestBody/searchScope.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_search.json
diff --git a/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson b/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
deleted file mode 100644
index ad886ca..0000000
--- a/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
+++ /dev/null
@@ -1,2 +0,0 @@
-{ "create" : { "_id": "$itemId" }}
-{ "itemId": "$itemId", "itemType": "profileAlias", "profileID": "$itemId", "scope": null, "clientID": "defaultClientId", "creationTime": "$migrationTime", "modifiedTime": "$migrationTime"}
diff --git a/tools/shell-commands/src/main/resources/requestBody/bulkSaveScope.ndjson b/tools/shell-commands/src/main/resources/requestBody/bulkSaveScope.ndjson
deleted file mode 100644
index 32f9bb0..0000000
--- a/tools/shell-commands/src/main/resources/requestBody/bulkSaveScope.ndjson
+++ /dev/null
@@ -1,2 +0,0 @@
-{"index": {"_id": "$scope"}}
-{"itemId": "$scope", "itemType": "scope", "metadata": {"id": "$scope" }}
diff --git a/tools/shell-commands/src/main/resources/requestBody/copyValueScopeToSourceId.json b/tools/shell-commands/src/main/resources/requestBody/copyValueScopeToSourceId.json
deleted file mode 100644
index 8d9a1dd..0000000
--- a/tools/shell-commands/src/main/resources/requestBody/copyValueScopeToSourceId.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "script": {
- "source": "ctx._source.sourceId = ctx._source.scope",
- "lang": "painless"
- }
-}
