| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License |
| */ |
| package org.apache.unomi.itests; |
| |
| import org.apache.unomi.api.Metadata; |
| import org.apache.unomi.api.PartialList; |
| import org.apache.unomi.api.Profile; |
| import org.apache.unomi.api.PropertyType; |
| import org.apache.unomi.api.services.ProfileService; |
| import org.apache.unomi.router.api.ImportConfiguration; |
| import org.apache.unomi.router.api.RouterConstants; |
| import org.apache.unomi.router.api.RouterUtils; |
| import org.apache.unomi.router.api.services.ImportExportConfigurationService; |
| import org.junit.Assert; |
| import org.junit.Test; |
| import org.junit.runner.RunWith; |
| import org.ops4j.pax.exam.junit.PaxExam; |
| import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; |
| import org.ops4j.pax.exam.spi.reactors.PerSuite; |
| import org.ops4j.pax.exam.util.Filter; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import javax.inject.Inject; |
| import java.io.File; |
| import java.io.IOException; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.Map; |
| import java.util.Objects; |
| |
| /** |
| * Created by amidani on 09/08/2017. |
| */ |
| @RunWith(PaxExam.class) |
| @ExamReactorStrategy(PerSuite.class) |
| public class ProfileImportSurfersIT extends BaseIT { |
| private Logger logger = LoggerFactory.getLogger(ProfileImportSurfersIT.class); |
| |
| @Inject @Filter(value="(configDiscriminator=IMPORT)", timeout = 600000) |
| protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService; |
| @Inject @Filter(timeout = 600000) |
| protected ProfileService profileService; |
| |
| @Test |
| public void testImportSurfers() throws IOException, InterruptedException { |
| |
| /*** Create Missing Properties ***/ |
| PropertyType propertyType = new PropertyType(new Metadata("integration", "alive", "Alive", "Is the person alive?")); |
| propertyType.setValueTypeId("boolean"); |
| propertyType.getMetadata().setSystemTags(Collections.singleton("basicProfileProperties")); |
| propertyType.setTarget("profiles"); |
| |
| profileService.setPropertyType(propertyType); |
| |
| keepTrying("Failed waiting for property type 'alive'", |
| () -> profileService.getPropertyType("alive"), |
| Objects::nonNull, |
| 1000, |
| 100); |
| |
| PropertyType propAlive = RouterUtils.getPropertyTypeById(profileService.getTargetPropertyTypes("profiles"), "alive"); |
| |
| Assert.assertNotNull("Lookup of alive property through profiles target has failed !", propAlive); |
| |
| /*** Surfers Test ***/ |
| ImportConfiguration importConfigSurfers = new ImportConfiguration(); |
| String itemId1 = "2-surfers-test"; |
| importConfigSurfers.setItemId(itemId1); |
| importConfigSurfers.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_RECURRENT); |
| importConfigSurfers.setMergingProperty("linkedInId"); |
| importConfigSurfers.setOverwriteExistingProfiles(true); |
| importConfigSurfers.setColumnSeparator(";"); |
| importConfigSurfers.setHasHeader(true); |
| importConfigSurfers.setHasDeleteColumn(true); |
| |
| Map<String, Integer> mappingSurfers = new HashMap<>(); |
| mappingSurfers.put("linkedInId", 0); |
| mappingSurfers.put("lastName", 1); |
| mappingSurfers.put("email", 2); |
| mappingSurfers.put("facebookId", 3); |
| mappingSurfers.put("gender", 4); |
| mappingSurfers.put("alive", 5); |
| mappingSurfers.put("city", 6); |
| |
| importConfigSurfers.getProperties().put("mapping", mappingSurfers); |
| File importSurfersFile = new File("data/tmp/recurrent_import/"); |
| importConfigSurfers.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=2-surfers-test.csv&consumer.delay=10m&move=.done"); |
| importConfigSurfers.setActive(true); |
| |
| importConfigurationService.save(importConfigSurfers, true); |
| |
| logger.info("ProfileImportSurfersIT setup successfully."); |
| |
| //Wait for data to be processed |
| keepTrying("Failed waiting for surfers initial import to complete", ()->profileService.findProfilesByPropertyValue("properties.city", "surfersCity", 0, 50, null), (p)->p.getTotalSize() == 34, 1000, 100); |
| |
| keepTrying("Failed waiting for import configurations list with 1 item", |
| () -> importConfigurationService.getAll(), |
| (list) -> Objects.nonNull(list) && list.size() == 1, |
| 1000, |
| 100); |
| |
| //Profile not to delete |
| PartialList<Profile> jordyProfile = profileService.findProfilesByPropertyValue("properties.email", "jordy@smith.com", 0, 10, null); |
| Assert.assertEquals(1, jordyProfile.getList().size()); |
| Assert.assertNotNull(jordyProfile.get(0)); |
| Assert.assertEquals("1", jordyProfile.get(0).getProperty("linkedInId")); |
| Assert.assertEquals("Jordy Smith", jordyProfile.get(0).getProperty("lastName")); |
| Assert.assertEquals("999", jordyProfile.get(0).getProperty("facebookId")); |
| Assert.assertEquals("male", jordyProfile.get(0).getProperty("gender")); |
| Assert.assertTrue((Boolean) jordyProfile.get(0).getProperty("alive")); |
| |
| //Profile to delete |
| PartialList<Profile> paulineProfile = profileService.findProfilesByPropertyValue("properties.lastName", "Pauline Ado", 0, 10, null); |
| Assert.assertEquals(0, paulineProfile.getList().size()); |
| |
| //Check import config status |
| ImportConfiguration importConfiguration = importConfigurationService.load(itemId1); |
| Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus()); |
| Assert.assertEquals(1, importConfiguration.getExecutions().size()); |
| importConfigurationService.delete(itemId1); |
| |
| /*** Surfers Test OVERWRITE ***/ |
| ImportConfiguration importConfigSurfersOverwrite = new ImportConfiguration(); |
| String itemId2 = "3-surfers-overwrite-test"; |
| importConfigSurfersOverwrite.setItemId(itemId2); |
| importConfigSurfersOverwrite.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_RECURRENT); |
| importConfigSurfersOverwrite.setMergingProperty("linkedInId"); |
| importConfigSurfersOverwrite.setOverwriteExistingProfiles(true); |
| importConfigSurfersOverwrite.setColumnSeparator(";"); |
| importConfigSurfersOverwrite.setHasHeader(true); |
| importConfigSurfersOverwrite.setHasDeleteColumn(true); |
| |
| importConfigSurfersOverwrite.getProperties().put("mapping", mappingSurfers); |
| importConfigSurfersOverwrite.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=3-surfers-overwrite-test.csv&consumer.delay=10m&move=.done"); |
| importConfigSurfersOverwrite.setActive(true); |
| |
| importConfigurationService.save(importConfigSurfersOverwrite, true); |
| |
| logger.info("ProfileImportSurfersOverwriteIT setup successfully."); |
| |
| //Wait for data to be processed |
| keepTrying("Failed waiting for surfers overwrite import to complete", ()->profileService.findProfilesByPropertyValue("properties.city", "surfersCity", 0, 50, null), (p)->p.getTotalSize() == 36, 1000, 100); |
| |
| keepTrying("Failed waiting for import configurations list with 1 item", |
| () -> importConfigurationService.getAll(), |
| (list) -> Objects.nonNull(list) && list.size() == 1, |
| 1000, |
| 100); |
| |
| //Profile not to delete |
| PartialList<Profile> aliveProfiles = profileService.findProfilesByPropertyValue("properties.alive", "true", 0, 50, null); |
| PartialList<Profile> deadProfiles = profileService.findProfilesByPropertyValue("properties.alive", "false", 0, 50, null); |
| |
| Assert.assertEquals(0, aliveProfiles.getList().size()); |
| Assert.assertEquals(36, deadProfiles.getList().size()); |
| |
| //Profile to delete = false, was to delete |
| PartialList<Profile> paulineProfileOverwrite = profileService.findProfilesByPropertyValue("properties.lastName", "Pauline Ado", 0, 10, null); |
| Assert.assertEquals(1, paulineProfileOverwrite.getList().size()); |
| importConfigurationService.delete(itemId2); |
| |
| /*** Surfers Delete Test ***/ |
| |
| ImportConfiguration importConfigSurfersDelete = new ImportConfiguration(); |
| String itemId3 = "4-surfers-delete-test"; |
| importConfigSurfersDelete.setItemId(itemId3); |
| importConfigSurfersDelete.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_RECURRENT); |
| importConfigSurfersDelete.setMergingProperty("linkedInId"); |
| importConfigSurfersDelete.setOverwriteExistingProfiles(true); |
| importConfigSurfersDelete.setColumnSeparator(";"); |
| importConfigSurfersDelete.setHasHeader(true); |
| importConfigSurfersDelete.setHasDeleteColumn(true); |
| |
| importConfigSurfersDelete.getProperties().put("mapping", mappingSurfers); |
| |
| importConfigSurfersDelete.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=4-surfers-delete-test.csv&consumer.delay=10m&move=.done"); |
| importConfigSurfersDelete.setActive(true); |
| |
| importConfigurationService.save(importConfigSurfersDelete, true); |
| |
| logger.info("ProfileImportSurfersDeleteIT setup successfully."); |
| |
| //Wait for data to be processed |
| keepTrying("Failed waiting for surfers delete import to complete", ()->profileService.findProfilesByPropertyValue("properties.city", "surfersCity", 0, 50, null), (p)->p.getTotalSize() == 0, 1000, 100); |
| |
| keepTrying("Failed waiting for import configurations list with 1 item", |
| () -> importConfigurationService.getAll(), |
| (list) -> Objects.nonNull(list) && list.size() == 1, |
| 1000, |
| 100); |
| |
| PartialList<Profile> jordyProfileDelete = profileService.findProfilesByPropertyValue("properties.email", "jordy@smith.com", 0, 10, null); |
| Assert.assertEquals(0, jordyProfileDelete.getList().size()); |
| |
| importConfigurationService.delete(itemId3); |
| } |
| } |