| <!DOCTYPE html><html><head><title>Multiple Events and Multiple Algorithms</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta class="swiftype" name="title" data-type="string" content="Multiple Events and Multiple Algorithms"/><link rel="canonical" href="https://docs.prediction.io/templates/similarproduct/multi-events-multi-algos/"/><link href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" rel="stylesheet"/><link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"/><link href="/stylesheets/application-3598c7d7.css" rel="stylesheet" type="text/css"/><script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async: true });}catch(e){}</script></head><body><div id="global"><header><div class="container" id="header-wrapper"><div class="row"><div class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a href="#"></a><a href="http://predictionio.incubator.apache.org/"><img alt="PredictionIO" id="logo" src="/images/logos/logo-ee2b9bb3.png"/></a></div><div id="menu-wrapper"><div id="header-nav-options-wrapper"><ul><li><a href="/">Install & Doc</a></li> <li><a href="/support">Support</a></li> </ul></div><div id="pill-wrapper"><a class="pill left" href="//templates.prediction.io/">TEMPLATES</a> <a class="pill right" href="//github.com/apache/incubator-predictionio/">OPEN SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md hidden-lg" src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div id="search-bar-row-wrapper"><div class="container-fluid" id="search-bar-row"><div class="row"><div class="col-md-9 col-sm-11 col-xs-11"><div class="hidden-md hidden-lg" id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Multiple Events and Multiple Algorithms</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img id="left-menu-indicator" src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3 col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form class="search-form"><img class="search-box-toggler hidden-xs hidden-sm" src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img src="/images/icons/search-glass-704bd4ff.png"/><input type="text" id="st-search-input" class="st-search-input" placeholder="Search Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg" src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div class="mobile-left-menu-toggler hidden-md hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div class="row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO (incubating) Documentation</span></a><ul><li class="level-2"><a class="final" href="/"><span>Welcome to Apache PredictionIO (incubating)</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Started</span></a><ul><li class="level-2"><a class="final" href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a class="final" href="/install/"><span>Installing Apache PredictionIO (incubating)</span></a></li><li class="level-2"><a class="final" href="/start/download/"><span>Downloading an Engine Template</span></a></li><li class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your First Engine</span></a></li><li class="level-2"><a class="final" href="/start/customize/"><span>Customizing the Engine</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Integrating with Your App</span></a><ul><li class="level-2"><a class="final" href="/appintegration/"><span>App Integration Overview</span></a></li><li class="level-2"><a class="expandible" href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/php/"><span>PHP SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web Service</span></a></li><li class="level-2"><a class="final" href="/cli/#engine-commands"><span>Engine Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting Engine Parameters</span></a></li><li class="level-2"><a class="final" href="/deploy/enginevariants/"><span>Deploying Multiple Engine Variants</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Customizing an Engine</span></a><ul><li class="level-2"><a class="final" href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li class="level-2"><a class="final" href="/customize/troubleshooting/"><span>Troubleshooting Engine Development</span></a></li><li class="level-2"><a class="final" href="/api/current/#package"><span>Engine Scala APIs</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Collecting and Analyzing Data</span></a><ul><li class="level-2"><a class="final" href="/datacollection/"><span>Event Server Overview</span></a></li><li class="level-2"><a class="final" href="/cli/#event-server-commands"><span>Event Server Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventapi/"><span>Collecting Data with REST/SDKs</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li class="level-2"><a class="final" href="/datacollection/webhooks/"><span>Unifying Multichannel Data with Webhooks</span></a></li><li class="level-2"><a class="final" href="/datacollection/channel/"><span>Channel</span></a></li><li class="level-2"><a class="final" href="/datacollection/batchimport/"><span>Importing Data in Batch</span></a></li><li class="level-2"><a class="final" href="/datacollection/analytics/"><span>Using Analytics Tools</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Choosing an Algorithm(s)</span></a><ul><li class="level-2"><a class="final" href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to Another Algorithm</span></a></li><li class="level-2"><a class="final" href="/algorithm/multiple/"><span>Combining Multiple Algorithms</span></a></li><li class="level-2"><a class="final" href="/algorithm/custom/"><span>Adding Your Own Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a class="final" href="/evaluation/"><span>Overview</span></a></li><li class="level-2"><a class="final" href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li class="level-2"><a class="final" href="/evaluation/evaluationdashboard/"><span>Evaluation Dashboard</span></a></li><li class="level-2"><a class="final" href="/evaluation/metricchoose/"><span>Choosing Evaluation Metrics</span></a></li><li class="level-2"><a class="final" href="/evaluation/metricbuild/"><span>Building Evaluation Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>System Architecture</span></a><ul><li class="level-2"><a class="final" href="/system/"><span>Architecture Overview</span></a></li><li class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using Another Data Store</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li class="level-2"><a class="final" href="http://templates.prediction.io"><span>Browse</span></a></li><li class="level-2"><a class="final" href="/community/submit-template/"><span>Submit your Engine as a Template</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Demo Tutorials</span></a><ul><li class="level-2"><a class="final" href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li><li class="level-2"><a class="final" href="/demo/community/"><span>Community Contributed Demo</span></a></li><li class="level-2"><a class="final" href="/demo/textclassification/"><span>Text Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a class="final" href="/community/contribute-code/"><span>Contribute Code</span></a></li><li class="level-2"><a class="final" href="/community/contribute-documentation/"><span>Contribute Documentation</span></a></li><li class="level-2"><a class="final" href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li class="level-2"><a class="final" href="/community/contribute-webhook/"><span>Contribute a Webhook</span></a></li><li class="level-2"><a class="final" href="/community/projects/"><span>Community Projects</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Help</span></a><ul><li class="level-2"><a class="final" href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a class="final" href="/support/"><span>Community Support</span></a></li><li class="level-2"><a class="final" href="/support/#enterprise-support"><span>Enterprise Support</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final" href="/resources/intellij/"><span>Developing Engines with IntelliJ IDEA</span></a></li><li class="level-2"><a class="final" href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li class="level-2"><a class="final" href="/resources/glossary/"><span>Glossary</span></a></li></ul></li></ul></nav></div><div class="col-md-9 col-sm-12"><div class="content-header hidden-md hidden-lg"><div id="page-title"><h1>Multiple Events and Multiple Algorithms</h1></div></div><div id="table-of-content-wrapper"><a id="edit-page-link" href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/templates/similarproduct/multi-events-multi-algos.html.md.erb"><img src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div class="content-header hidden-sm hidden-xs"><div id="page-title"><h1>Multiple Events and Multiple Algorithms</h1></div></div><div class="content"><p>This is more advanced example, we recommend you go through the <a href="/templates/similarproduct/dase/">DASE</a> explanation first.</p><p>The <a href="/templates/similarproduct/dase/#algorithm">default algorithm described in DASE</a> uses user-to-item view events as training data. However, your application may have more than one type of events which you want to take into account, such as buy, rate and like events. One way to incorporate other types of events to improve the system is to add another algorithm to process these events, build a separated model and then combine the outputs of multiple algorithms during Serving.</p><p>In this example, we will add another algorithm to process like/dislike events. The final PredictedResults will be the combined outputs of both algorithms.</p><div class="alert-message note"><p>This is just one of the ways to handle mutliple types of events. We use this use case to demonstrate how one can build an engine with multiple algorithms. You may also build one single algorithm which takes different events into account without using multiple algorithms.</p></div><p>This example will demonstrate the following:</p> <ul> <li>Read multiple types of events</li> <li>Use positive and negative implicit events such as like and dislike with MLlib ALS algorithm</li> <li>Integrate multiple algorithms into one engine</li> </ul> <p>The complete source code of this examlpe can be found in <code>examples/scala-parallel-similarproduct-multi</code>.</p><div class="alert-message info"><p>Require PredictionIO version >= 0.8.6 to run this example.</p></div><h3 id='step-1.-read-"like"-and-"dislike"-events-as-trainingdata' class='header-anchors'>Step 1. Read "like" and "dislike" events as TrainingData</h3><p>Modify the following in DataSource.scala:</p> <ul> <li>In addiiton to the original <code>ViewEvent</code> class, add a new class <code>LikeEvent</code> which has a boolean <code>like</code> field to represent it's like or dislike event.</li> <li>Add a new field <code>likeEvents</code> into <code>TrainingData</code> class to store the <code>RDD[LikeEvents]</code>.</li> <li>Modidy DataSource's <code>read()</code> function to read "like" and "dislike" events from the Event Store.</li> </ul> <p>The modification is shown below:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 |
| 2 |
| 3 |
| 4 |
| 5 |
| 6 |
| 7 |
| 8 |
| 9 |
| 10 |
| 11 |
| 12 |
| 13 |
| 14 |
| 15 |
| 16 |
| 17 |
| 18 |
| 19 |
| 20 |
| 21 |
| 22 |
| 23 |
| 24 |
| 25 |
| 26 |
| 27 |
| 28 |
| 29 |
| 30 |
| 31 |
| 32 |
| 33 |
| 34 |
| 35 |
| 36 |
| 37 |
| 38 |
| 39 |
| 40 |
| 41 |
| 42 |
| 43 |
| 44 |
| 45 |
| 46 |
| 47 |
| 48 |
| 49 |
| 50 |
| 51 |
| 52 |
| 53 |
| 54 |
| 55 |
| 56 |
| 57 |
| 58 |
| 59 |
| 60 |
| 61 |
| 62 |
| 63 |
| 64 |
| 65 |
| 66 |
| 67 |
| 68 |
| 69 |
| 70 |
| 71 |
| 72 |
| 73 |
| 74</pre></td><td class="code"><pre> |
| <span class="k">class</span> <span class="nc">DataSource</span><span class="o">(</span><span class="k">val</span> <span class="n">dsp</span><span class="k">:</span> <span class="kt">DataSourceParams</span><span class="o">)</span> |
| <span class="k">extends</span> <span class="nc">PDataSource</span><span class="o">[</span><span class="kt">TrainingData</span>, |
| <span class="kt">EmptyEvaluationInfo</span>, <span class="kt">Query</span>, <span class="kt">EmptyActualResult</span><span class="o">]</span> <span class="o">{</span> |
| |
| <span class="k">override</span> |
| <span class="k">def</span> <span class="n">readTraining</span><span class="o">(</span><span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span><span class="o">)</span><span class="k">:</span> <span class="kt">TrainingData</span> <span class="o">=</span> <span class="o">{</span> |
| |
| <span class="o">...</span> |
| |
| <span class="c1">// get all "user" "view" "item" events |
| </span> <span class="k">val</span> <span class="n">viewEventsRDD</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">ViewEvent</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> |
| |
| <span class="c1">// ADDED |
| </span> <span class="c1">// get all "user" "like" and "dislike" "item" events |
| </span> <span class="k">val</span> <span class="n">likeEventsRDD</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">LikeEvent</span><span class="o">]</span> <span class="k">=</span> <span class="n">eventsDb</span><span class="o">.</span><span class="n">find</span><span class="o">(</span> |
| <span class="n">appId</span> <span class="k">=</span> <span class="n">dsp</span><span class="o">.</span><span class="n">appId</span><span class="o">,</span> |
| <span class="n">entityType</span> <span class="k">=</span> <span class="nc">Some</span><span class="o">(</span><span class="s">"user"</span><span class="o">),</span> |
| <span class="n">eventNames</span> <span class="k">=</span> <span class="nc">Some</span><span class="o">(</span><span class="nc">List</span><span class="o">(</span><span class="s">"like"</span><span class="o">,</span> <span class="s">"dislike"</span><span class="o">)),</span> |
| <span class="c1">// targetEntityType is optional field of an event. |
| </span> <span class="n">targetEntityType</span> <span class="k">=</span> <span class="nc">Some</span><span class="o">(</span><span class="nc">Some</span><span class="o">(</span><span class="s">"item"</span><span class="o">)))(</span><span class="n">sc</span><span class="o">)</span> |
| <span class="c1">// eventsDb.find() returns RDD[Event] |
| </span> <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="n">event</span> <span class="k">=></span> |
| <span class="k">val</span> <span class="n">likeEvent</span> <span class="k">=</span> <span class="k">try</span> <span class="o">{</span> |
| <span class="n">event</span><span class="o">.</span><span class="n">event</span> <span class="k">match</span> <span class="o">{</span> |
| <span class="k">case</span> <span class="s">"like"</span> <span class="o">|</span> <span class="s">"dislike"</span> <span class="k">=></span> <span class="nc">LikeEvent</span><span class="o">(</span> |
| <span class="n">user</span> <span class="k">=</span> <span class="n">event</span><span class="o">.</span><span class="n">entityId</span><span class="o">,</span> |
| <span class="n">item</span> <span class="k">=</span> <span class="n">event</span><span class="o">.</span><span class="n">targetEntityId</span><span class="o">.</span><span class="n">get</span><span class="o">,</span> |
| <span class="n">t</span> <span class="k">=</span> <span class="n">event</span><span class="o">.</span><span class="n">eventTime</span><span class="o">.</span><span class="n">getMillis</span><span class="o">,</span> |
| <span class="n">like</span> <span class="k">=</span> <span class="o">(</span><span class="n">event</span><span class="o">.</span><span class="n">event</span> <span class="o">==</span> <span class="s">"like"</span><span class="o">))</span> |
| <span class="k">case</span> <span class="k">_</span> <span class="k">=></span> <span class="k">throw</span> <span class="k">new</span> <span class="nc">Exception</span><span class="o">(</span><span class="n">s</span><span class="s">"Unexpected event ${event} is read."</span><span class="o">)</span> |
| <span class="o">}</span> |
| <span class="o">}</span> <span class="k">catch</span> <span class="o">{</span> |
| <span class="k">case</span> <span class="n">e</span><span class="k">:</span> <span class="kt">Exception</span> <span class="o">=></span> <span class="o">{</span> |
| <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="o">(</span><span class="n">s</span><span class="s">"Cannot convert ${event} to LikeEvent."</span> <span class="o">+</span> |
| <span class="n">s</span><span class="s">" Exception: ${e}."</span><span class="o">)</span> |
| <span class="k">throw</span> <span class="n">e</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| <span class="n">likeEvent</span> |
| <span class="o">}</span> |
| |
| <span class="k">new</span> <span class="nc">TrainingData</span><span class="o">(</span> |
| <span class="n">users</span> <span class="k">=</span> <span class="n">usersRDD</span><span class="o">,</span> |
| <span class="n">items</span> <span class="k">=</span> <span class="n">itemsRDD</span><span class="o">,</span> |
| <span class="n">viewEvents</span> <span class="k">=</span> <span class="n">viewEventsRDD</span><span class="o">,</span> |
| <span class="n">likeEvents</span> <span class="k">=</span> <span class="n">likeEventsRDD</span> <span class="c1">// ADDED |
| </span> <span class="o">)</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| |
| <span class="o">...</span> |
| |
| <span class="k">case</span> <span class="k">class</span> <span class="nc">LikeEvent</span><span class="o">(</span> <span class="c1">// ADDED |
| </span> <span class="n">user</span><span class="k">:</span> <span class="kt">String</span><span class="o">,</span> |
| <span class="n">item</span><span class="k">:</span> <span class="kt">String</span><span class="o">,</span> |
| <span class="n">t</span><span class="k">:</span> <span class="kt">Long</span><span class="o">,</span> |
| <span class="n">like</span><span class="k">:</span> <span class="kt">Boolean</span> <span class="c1">// true: like. false: dislike |
| </span><span class="o">)</span> |
| |
| <span class="k">class</span> <span class="nc">TrainingData</span><span class="o">(</span> |
| <span class="k">val</span> <span class="n">users</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[(</span><span class="kt">String</span>, <span class="kt">User</span><span class="o">)],</span> |
| <span class="k">val</span> <span class="n">items</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[(</span><span class="kt">String</span>, <span class="kt">Item</span><span class="o">)],</span> |
| <span class="k">val</span> <span class="n">viewEvents</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">ViewEvent</span><span class="o">],</span> |
| <span class="k">val</span> <span class="n">likeEvents</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">LikeEvent</span><span class="o">]</span> <span class="c1">// ADDED |
| </span><span class="o">)</span> <span class="k">extends</span> <span class="nc">Serializable</span> <span class="o">{</span> |
| <span class="k">override</span> <span class="k">def</span> <span class="n">toString</span> <span class="k">=</span> <span class="o">{</span> |
| <span class="n">s</span><span class="s">"users: [${users.count()} (${users.take(2).toList}...)]"</span> <span class="o">+</span> |
| <span class="n">s</span><span class="s">"items: [${items.count()} (${items.take(2).toList}...)]"</span> <span class="o">+</span> |
| <span class="n">s</span><span class="s">"viewEvents: [${viewEvents.count()}] (${viewEvents.take(2).toList}...)"</span> <span class="o">+</span> |
| <span class="c1">// ADDED |
| </span> <span class="n">s</span><span class="s">"likeEvents: [${likeEvents.count()}] (${likeEvents.take(2).toList}...)"</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| </pre></td></tr></tbody></table> </div> <h3 id='step-2.-modify-preparator-and-prepareddata' class='header-anchors'>Step 2. Modify Preparator and PreparedData</h3><p>In Preparator.scala, simply pass the newly added <code>likeEvents</code> from <code>TrainingData</code> to <code>PreparedData</code>, as shown the code below:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 |
| 2 |
| 3 |
| 4 |
| 5 |
| 6 |
| 7 |
| 8 |
| 9 |
| 10 |
| 11 |
| 12 |
| 13 |
| 14 |
| 15 |
| 16 |
| 17 |
| 18 |
| 19 |
| 20</pre></td><td class="code"><pre> |
| <span class="k">class</span> <span class="nc">Preparator</span> |
| <span class="k">extends</span> <span class="nc">PPreparator</span><span class="o">[</span><span class="kt">TrainingData</span>, <span class="kt">PreparedData</span><span class="o">]</span> <span class="o">{</span> |
| |
| <span class="k">def</span> <span class="n">prepare</span><span class="o">(</span><span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span><span class="o">,</span> <span class="n">trainingData</span><span class="k">:</span> <span class="kt">TrainingData</span><span class="o">)</span><span class="k">:</span> <span class="kt">PreparedData</span> <span class="o">=</span> <span class="o">{</span> |
| <span class="k">new</span> <span class="nc">PreparedData</span><span class="o">(</span> |
| <span class="n">users</span> <span class="k">=</span> <span class="n">trainingData</span><span class="o">.</span><span class="n">users</span><span class="o">,</span> |
| <span class="n">items</span> <span class="k">=</span> <span class="n">trainingData</span><span class="o">.</span><span class="n">items</span><span class="o">,</span> |
| <span class="n">viewEvents</span> <span class="k">=</span> <span class="n">trainingData</span><span class="o">.</span><span class="n">viewEvents</span><span class="o">,</span> |
| <span class="n">likeEvents</span> <span class="k">=</span> <span class="n">trainingData</span><span class="o">.</span><span class="n">likeEvents</span><span class="o">)</span> <span class="c1">// ADDED |
| </span> <span class="o">}</span> |
| <span class="o">}</span> |
| |
| <span class="k">class</span> <span class="nc">PreparedData</span><span class="o">(</span> |
| <span class="k">val</span> <span class="n">users</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[(</span><span class="kt">String</span>, <span class="kt">User</span><span class="o">)],</span> |
| <span class="k">val</span> <span class="n">items</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[(</span><span class="kt">String</span>, <span class="kt">Item</span><span class="o">)],</span> |
| <span class="k">val</span> <span class="n">viewEvents</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">ViewEvent</span><span class="o">],</span> |
| <span class="k">val</span> <span class="n">likeEvents</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">LikeEvent</span><span class="o">]</span> <span class="c1">// ADDED |
| </span><span class="o">)</span> <span class="k">extends</span> <span class="nc">Serializable</span> |
| |
| </pre></td></tr></tbody></table> </div> <h3 id='step-3.-add-a-new-algorithm-to-train-model-with-<code>likeevents</code>' class='header-anchors' >Step 3. Add a new algorithm to train model with <code>likeEvents</code></h3><p>For demonstration purpose, we also use MLlib ALS to train model with <code>likeEvents</code> and hence the new algorithm class will share many common logic of the original algorithm. The only difference will be the <code>train()</code> function - the original one trains model with <code>viewEvents</code> while the new one uses <code>likeEvents</code>. In this case, we can simply create a new algorithm which extends the original <code>ALSAlgorithm</code> class and override the <code>train()</code> function.</p><div class="alert-message note"><p>You may also create a completely new algorithm to process <code>likeEvents</code> or other types of events without extending any existing algorithm, or simply modifying the same algorithm to take new events into account.</p></div><h4 id='using-positive-and-negative-implicit-events-(without-explicit-rating)-with-mllib-als' class='header-anchors'>Using positive and negative implicit events (without explicit rating) with MLlib ALS</h4><p>In the original <code>ALSAlgorithm</code>, the <code>train()</code> function calculates the number of times that the user has viewed the same item and then map it to <code>MLlibRating</code> object. However, like/dislike event is boolean and one time preference, so it doesn't make sense to aggregate the events if the user has multiple like/dislike events on the same item. However, a user may like an item and change her mind to dislike the same item later, or vice versa. In this case, we could simply use the latest like or dislike event of the user for the same item.</p><p>In addition, MLlib ALS can handle negative preference with <code>ALS.trainImplicit()</code>. Hence, we can map a dislike to rating of -1 and like to 1.</p><div class="alert-message info"><p>Negative preference does not work if use <code>ALS.train()</code> instead, which is for explicit rating such as "rate" event.</p></div><p>In summary, this new <code>LikeAlgorithm</code> does the following:</p> <ul> <li>Extends original <code>ALSAlgorithm</code> class</li> <li>Override <code>train()</code> to process the <code>likeEvents</code> in <code>PreparedData</code></li> <li>Use the latest event if the user likes/dislikes the same item multiple times</li> <li>Map dislike to a <code>MLlibRating</code> object with rating of -1 and like to rating of 1</li> <li>Use the <code>MLlibRating</code> to train the <code>ALSModel</code> in the same way as the original <code>ALSAlgorithm</code></li> <li>The <code>predict()</code> function is the same as the original <code>ALSAlgorithm</code></li> </ul> <p>It is shown in the code below:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 |
| 2 |
| 3 |
| 4 |
| 5 |
| 6 |
| 7 |
| 8 |
| 9 |
| 10 |
| 11 |
| 12 |
| 13 |
| 14 |
| 15 |
| 16 |
| 17 |
| 18 |
| 19 |
| 20 |
| 21 |
| 22 |
| 23 |
| 24 |
| 25 |
| 26 |
| 27 |
| 28 |
| 29 |
| 30 |
| 31 |
| 32 |
| 33 |
| 34 |
| 35 |
| 36 |
| 37 |
| 38 |
| 39 |
| 40 |
| 41 |
| 42 |
| 43 |
| 44 |
| 45 |
| 46 |
| 47 |
| 48 |
| 49 |
| 50 |
| 51 |
| 52 |
| 53 |
| 54 |
| 55 |
| 56 |
| 57 |
| 58 |
| 59 |
| 60 |
| 61 |
| 62 |
| 63 |
| 64 |
| 65 |
| 66 |
| 67 |
| 68 |
| 69 |
| 70 |
| 71 |
| 72 |
| 73 |
| 74 |
| 75 |
| 76 |
| 77 |
| 78 |
| 79 |
| 80 |
| 81 |
| 82 |
| 83 |
| 84 |
| 85 |
| 86 |
| 87 |
| 88 |
| 89 |
| 90 |
| 91 |
| 92 |
| 93 |
| 94 |
| 95 |
| 96 |
| 97 |
| 98 |
| 99 |
| 100 |
| 101 |
| 102 |
| 103</pre></td><td class="code"><pre><span class="k">package</span> <span class="nn">org.template.similarproduct</span> |
| |
| <span class="k">import</span> <span class="nn">io.prediction.data.storage.BiMap</span> |
| |
| <span class="k">import</span> <span class="nn">org.apache.spark.SparkContext</span> |
| <span class="k">import</span> <span class="nn">org.apache.spark.SparkContext._</span> |
| <span class="k">import</span> <span class="nn">org.apache.spark.rdd.RDD</span> |
| <span class="k">import</span> <span class="nn">org.apache.spark.mllib.recommendation.ALS</span> |
| <span class="k">import</span> <span class="nn">org.apache.spark.mllib.recommendation.</span><span class="o">{</span><span class="nc">Rating</span> <span class="k">=></span> <span class="nc">MLlibRating</span><span class="o">}</span> |
| |
| <span class="k">import</span> <span class="nn">grizzled.slf4j.Logger</span> |
| |
| <span class="c1">// ADDED |
| // Extend original ALSAlgorithm and override train() function to handle |
| // like and dislike events |
| </span><span class="k">class</span> <span class="nc">LikeAlgorithm</span><span class="o">(</span><span class="n">ap</span><span class="k">:</span> <span class="kt">ALSAlgorithmParams</span><span class="o">)</span> <span class="k">extends</span> <span class="nc">ALSAlgorithm</span><span class="o">(</span><span class="n">ap</span><span class="o">)</span> <span class="o">{</span> |
| |
| <span class="nd">@transient</span> <span class="k">lazy</span> <span class="k">override</span> <span class="k">val</span> <span class="n">logger</span> <span class="k">=</span> <span class="nc">Logger</span><span class="o">[</span><span class="kt">this.</span><span class="k">type</span><span class="o">]</span> |
| |
| <span class="k">override</span> |
| <span class="k">def</span> <span class="n">train</span><span class="o">(</span><span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span><span class="o">,</span> <span class="n">data</span><span class="k">:</span> <span class="kt">PreparedData</span><span class="o">)</span><span class="k">:</span> <span class="kt">ALSModel</span> <span class="o">=</span> <span class="o">{</span> |
| <span class="n">require</span><span class="o">(!</span><span class="n">data</span><span class="o">.</span><span class="n">likeEvents</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">1</span><span class="o">).</span><span class="n">isEmpty</span><span class="o">,</span> |
| <span class="n">s</span><span class="s">"likeEvents in PreparedData cannot be empty."</span> <span class="o">+</span> |
| <span class="s">" Please check if DataSource generates TrainingData"</span> <span class="o">+</span> |
| <span class="s">" and Preprator generates PreparedData correctly."</span><span class="o">)</span> |
| <span class="n">require</span><span class="o">(!</span><span class="n">data</span><span class="o">.</span><span class="n">users</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">1</span><span class="o">).</span><span class="n">isEmpty</span><span class="o">,</span> |
| <span class="n">s</span><span class="s">"users in PreparedData cannot be empty."</span> <span class="o">+</span> |
| <span class="s">" Please check if DataSource generates TrainingData"</span> <span class="o">+</span> |
| <span class="s">" and Preprator generates PreparedData correctly."</span><span class="o">)</span> |
| <span class="n">require</span><span class="o">(!</span><span class="n">data</span><span class="o">.</span><span class="n">items</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">1</span><span class="o">).</span><span class="n">isEmpty</span><span class="o">,</span> |
| <span class="n">s</span><span class="s">"items in PreparedData cannot be empty."</span> <span class="o">+</span> |
| <span class="s">" Please check if DataSource generates TrainingData"</span> <span class="o">+</span> |
| <span class="s">" and Preprator generates PreparedData correctly."</span><span class="o">)</span> |
| <span class="c1">// create User and item's String ID to integer index BiMap |
| </span> <span class="k">val</span> <span class="n">userStringIntMap</span> <span class="k">=</span> <span class="nc">BiMap</span><span class="o">.</span><span class="n">stringInt</span><span class="o">(</span><span class="n">data</span><span class="o">.</span><span class="n">users</span><span class="o">.</span><span class="n">keys</span><span class="o">)</span> |
| <span class="k">val</span> <span class="n">itemStringIntMap</span> <span class="k">=</span> <span class="nc">BiMap</span><span class="o">.</span><span class="n">stringInt</span><span class="o">(</span><span class="n">data</span><span class="o">.</span><span class="n">items</span><span class="o">.</span><span class="n">keys</span><span class="o">)</span> |
| |
| <span class="c1">// collect Item as Map and convert ID to Int index |
| </span> <span class="k">val</span> <span class="n">items</span><span class="k">:</span> <span class="kt">Map</span><span class="o">[</span><span class="kt">Int</span>, <span class="kt">Item</span><span class="o">]</span> <span class="k">=</span> <span class="n">data</span><span class="o">.</span><span class="n">items</span><span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="k">case</span> <span class="o">(</span><span class="n">id</span><span class="o">,</span> <span class="n">item</span><span class="o">)</span> <span class="k">=></span> |
| <span class="o">(</span><span class="n">itemStringIntMap</span><span class="o">(</span><span class="n">id</span><span class="o">),</span> <span class="n">item</span><span class="o">)</span> |
| <span class="o">}.</span><span class="n">collectAsMap</span><span class="o">.</span><span class="n">toMap</span> |
| |
| <span class="k">val</span> <span class="n">mllibRatings</span> <span class="k">=</span> <span class="n">data</span><span class="o">.</span><span class="n">likeEvents</span> |
| <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="n">r</span> <span class="k">=></span> |
| <span class="c1">// Convert user and item String IDs to Int index for MLlib |
| </span> <span class="k">val</span> <span class="n">uindex</span> <span class="k">=</span> <span class="n">userStringIntMap</span><span class="o">.</span><span class="n">getOrElse</span><span class="o">(</span><span class="n">r</span><span class="o">.</span><span class="n">user</span><span class="o">,</span> <span class="o">-</span><span class="mi">1</span><span class="o">)</span> |
| <span class="k">val</span> <span class="n">iindex</span> <span class="k">=</span> <span class="n">itemStringIntMap</span><span class="o">.</span><span class="n">getOrElse</span><span class="o">(</span><span class="n">r</span><span class="o">.</span><span class="n">item</span><span class="o">,</span> <span class="o">-</span><span class="mi">1</span><span class="o">)</span> |
| |
| <span class="k">if</span> <span class="o">(</span><span class="n">uindex</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="o">)</span> |
| <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="o">(</span><span class="n">s</span><span class="s">"Couldn't convert nonexistent user ID ${r.user}"</span> |
| <span class="o">+</span> <span class="s">" to Int index."</span><span class="o">)</span> |
| |
| <span class="k">if</span> <span class="o">(</span><span class="n">iindex</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="o">)</span> |
| <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="o">(</span><span class="n">s</span><span class="s">"Couldn't convert nonexistent item ID ${r.item}"</span> |
| <span class="o">+</span> <span class="s">" to Int index."</span><span class="o">)</span> |
| |
| <span class="c1">// key is (uindex, iindex) tuple, value is (like, t) tuple |
| </span> <span class="o">((</span><span class="n">uindex</span><span class="o">,</span> <span class="n">iindex</span><span class="o">),</span> <span class="o">(</span><span class="n">r</span><span class="o">.</span><span class="n">like</span><span class="o">,</span> <span class="n">r</span><span class="o">.</span><span class="n">t</span><span class="o">))</span> |
| <span class="o">}.</span><span class="n">filter</span> <span class="o">{</span> <span class="k">case</span> <span class="o">((</span><span class="n">u</span><span class="o">,</span> <span class="n">i</span><span class="o">),</span> <span class="n">v</span><span class="o">)</span> <span class="k">=></span> |
| <span class="c1">//val = d |
| </span> <span class="c1">// keep events with valid user and item index |
| </span> <span class="o">(</span><span class="n">u</span> <span class="o">!=</span> <span class="o">-</span><span class="mi">1</span><span class="o">)</span> <span class="o">&&</span> <span class="o">(</span><span class="n">i</span> <span class="o">!=</span> <span class="o">-</span><span class="mi">1</span><span class="o">)</span> |
| <span class="o">}.</span><span class="n">reduceByKey</span> <span class="o">{</span> <span class="k">case</span> <span class="o">(</span><span class="n">v1</span><span class="o">,</span> <span class="n">v2</span><span class="o">)</span> <span class="k">=></span> <span class="c1">// MODIFIED |
| </span> <span class="c1">// An user may like an item and change to dislike it later, |
| </span> <span class="c1">// or vice versa. Use the latest value for this case. |
| </span> <span class="k">val</span> <span class="o">(</span><span class="n">like1</span><span class="o">,</span> <span class="n">t1</span><span class="o">)</span> <span class="k">=</span> <span class="n">v1</span> |
| <span class="k">val</span> <span class="o">(</span><span class="n">like2</span><span class="o">,</span> <span class="n">t2</span><span class="o">)</span> <span class="k">=</span> <span class="n">v2</span> |
| <span class="c1">// keep the latest value |
| </span> <span class="k">if</span> <span class="o">(</span><span class="n">t1</span> <span class="o">></span> <span class="n">t2</span><span class="o">)</span> <span class="n">v1</span> <span class="k">else</span> <span class="n">v2</span> |
| <span class="o">}.</span><span class="n">map</span> <span class="o">{</span> <span class="k">case</span> <span class="o">((</span><span class="n">u</span><span class="o">,</span> <span class="n">i</span><span class="o">),</span> <span class="o">(</span><span class="n">like</span><span class="o">,</span> <span class="n">t</span><span class="o">))</span> <span class="k">=></span> <span class="c1">// MODIFIED |
| </span> <span class="c1">// With ALS.trainImplicit(), we can use negative value to indicate |
| </span> <span class="c1">// nagative siginal (ie. dislike) |
| </span> <span class="k">val</span> <span class="n">r</span> <span class="k">=</span> <span class="k">if</span> <span class="o">(</span><span class="n">like</span><span class="o">)</span> <span class="mi">1</span> <span class="k">else</span> <span class="o">-</span><span class="mi">1</span> |
| <span class="c1">// MLlibRating requires integer index for user and item |
| </span> <span class="nc">MLlibRating</span><span class="o">(</span><span class="n">u</span><span class="o">,</span> <span class="n">i</span><span class="o">,</span> <span class="n">r</span><span class="o">)</span> |
| <span class="o">}</span> |
| <span class="o">.</span><span class="n">cache</span><span class="o">()</span> |
| |
| <span class="c1">// MLLib ALS cannot handle empty training data. |
| </span> <span class="n">require</span><span class="o">(!</span><span class="n">mllibRatings</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">1</span><span class="o">).</span><span class="n">isEmpty</span><span class="o">,</span> |
| <span class="n">s</span><span class="s">"mllibRatings cannot be empty."</span> <span class="o">+</span> |
| <span class="s">" Please check if your events contain valid user and item ID."</span><span class="o">)</span> |
| <span class="c1">// seed for MLlib ALS |
| </span> <span class="k">val</span> <span class="n">seed</span> <span class="k">=</span> <span class="n">ap</span><span class="o">.</span><span class="n">seed</span><span class="o">.</span><span class="n">getOrElse</span><span class="o">(</span><span class="nc">System</span><span class="o">.</span><span class="n">nanoTime</span><span class="o">)</span> |
| |
| <span class="k">val</span> <span class="n">m</span> <span class="k">=</span> <span class="nc">ALS</span><span class="o">.</span><span class="n">trainImplicit</span><span class="o">(</span> |
| <span class="n">ratings</span> <span class="k">=</span> <span class="n">mllibRatings</span><span class="o">,</span> |
| <span class="n">rank</span> <span class="k">=</span> <span class="n">ap</span><span class="o">.</span><span class="n">rank</span><span class="o">,</span> |
| <span class="n">iterations</span> <span class="k">=</span> <span class="n">ap</span><span class="o">.</span><span class="n">numIterations</span><span class="o">,</span> |
| <span class="n">lambda</span> <span class="k">=</span> <span class="n">ap</span><span class="o">.</span><span class="n">lambda</span><span class="o">,</span> |
| <span class="n">blocks</span> <span class="k">=</span> <span class="o">-</span><span class="mi">1</span><span class="o">,</span> |
| <span class="n">alpha</span> <span class="k">=</span> <span class="mf">1.0</span><span class="o">,</span> |
| <span class="n">seed</span> <span class="k">=</span> <span class="n">seed</span><span class="o">)</span> |
| |
| <span class="k">new</span> <span class="nc">ALSModel</span><span class="o">(</span> |
| <span class="n">productFeatures</span> <span class="k">=</span> <span class="n">m</span><span class="o">.</span><span class="n">productFeatures</span><span class="o">,</span> |
| <span class="n">itemStringIntMap</span> <span class="k">=</span> <span class="n">itemStringIntMap</span><span class="o">,</span> |
| <span class="n">items</span> <span class="k">=</span> <span class="n">items</span> |
| <span class="o">)</span> |
| <span class="o">}</span> |
| |
| <span class="o">}</span> |
| |
| </pre></td></tr></tbody></table> </div> <h3 id='step-4.-modify-serving-to-combine-multiple-algorithms'-outputs' class='header-anchors'>Step 4. Modify Serving to combine multiple algorithms' outputs</h3><p>When the engine is deployed, the Query is sent to all algorithms of the engine. The PredictedResults returned by all algoritms are passed to Serving component for further processing, as you could see that the argument <code>predictedResults</code> of the <code>serve()</code> function is type of <code>Seq[PredictedResult]</code>.</p><p>In this example, the <code>serve()</code> function at first standardizes the PredictedResults of each algorithm so that we can combine the scores of multiple algorithms by adding the scores of the same item. Then we can take the top N items as defined in <code>query</code>.</p><p>Modify Serving.scala as shown below:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 |
| 2 |
| 3 |
| 4 |
| 5 |
| 6 |
| 7 |
| 8 |
| 9 |
| 10 |
| 11 |
| 12 |
| 13 |
| 14 |
| 15 |
| 16 |
| 17 |
| 18 |
| 19 |
| 20 |
| 21 |
| 22 |
| 23 |
| 24 |
| 25 |
| 26 |
| 27 |
| 28 |
| 29 |
| 30 |
| 31 |
| 32 |
| 33 |
| 34 |
| 35 |
| 36 |
| 37 |
| 38 |
| 39 |
| 40 |
| 41 |
| 42 |
| 43 |
| 44 |
| 45 |
| 46 |
| 47 |
| 48</pre></td><td class="code"><pre> |
| <span class="k">class</span> <span class="nc">Serving</span> |
| <span class="k">extends</span> <span class="nc">LServing</span><span class="o">[</span><span class="kt">Query</span>, <span class="kt">PredictedResult</span><span class="o">]</span> <span class="o">{</span> |
| |
| <span class="k">override</span> |
| <span class="k">def</span> <span class="n">serve</span><span class="o">(</span><span class="n">query</span><span class="k">:</span> <span class="kt">Query</span><span class="o">,</span> |
| <span class="n">predictedResults</span><span class="k">:</span> <span class="kt">Seq</span><span class="o">[</span><span class="kt">PredictedResult</span><span class="o">])</span><span class="k">:</span> <span class="kt">PredictedResult</span> <span class="o">=</span> <span class="o">{</span> |
| |
| <span class="c1">// MODFIED |
| </span> <span class="k">val</span> <span class="n">standard</span><span class="k">:</span> <span class="kt">Seq</span><span class="o">[</span><span class="kt">Array</span><span class="o">[</span><span class="kt">ItemScore</span><span class="o">]]</span> <span class="k">=</span> <span class="k">if</span> <span class="o">(</span><span class="n">query</span><span class="o">.</span><span class="n">num</span> <span class="o">==</span> <span class="mi">1</span><span class="o">)</span> <span class="o">{</span> |
| <span class="c1">// if query 1 item, don't standardize |
| </span> <span class="n">predictedResults</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">itemScores</span><span class="o">)</span> |
| <span class="o">}</span> <span class="k">else</span> <span class="o">{</span> |
| <span class="c1">// Standardize the score before combine |
| </span> <span class="k">val</span> <span class="n">mvList</span><span class="k">:</span> <span class="kt">Seq</span><span class="o">[</span><span class="kt">MeanAndVariance</span><span class="o">]</span> <span class="k">=</span> <span class="n">predictedResults</span><span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="n">pr</span> <span class="k">=></span> |
| <span class="n">meanAndVariance</span><span class="o">(</span><span class="n">pr</span><span class="o">.</span><span class="n">itemScores</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">score</span><span class="o">))</span> |
| <span class="o">}</span> |
| |
| <span class="n">predictedResults</span><span class="o">.</span><span class="n">zipWithIndex</span> |
| <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="k">case</span> <span class="o">(</span><span class="n">pr</span><span class="o">,</span> <span class="n">i</span><span class="o">)</span> <span class="k">=></span> |
| <span class="n">pr</span><span class="o">.</span><span class="n">itemScores</span><span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="n">is</span> <span class="k">=></span> |
| <span class="c1">// standardize score (z-score) |
| </span> <span class="c1">// if standard deviation is 0 (when all items have the same score, |
| </span> <span class="c1">// meaning all items are ranked equally), return 0. |
| </span> <span class="k">val</span> <span class="n">score</span> <span class="k">=</span> <span class="k">if</span> <span class="o">(</span><span class="n">mvList</span><span class="o">(</span><span class="n">i</span><span class="o">).</span><span class="n">stdDev</span> <span class="o">==</span> <span class="mi">0</span><span class="o">)</span> <span class="o">{</span> |
| <span class="mi">0</span> |
| <span class="o">}</span> <span class="k">else</span> <span class="o">{</span> |
| <span class="o">(</span><span class="n">is</span><span class="o">.</span><span class="n">score</span> <span class="o">-</span> <span class="n">mvList</span><span class="o">(</span><span class="n">i</span><span class="o">).</span><span class="n">mean</span><span class="o">)</span> <span class="o">/</span> <span class="n">mvList</span><span class="o">(</span><span class="n">i</span><span class="o">).</span><span class="n">stdDev</span> |
| <span class="o">}</span> |
| |
| <span class="nc">ItemScore</span><span class="o">(</span><span class="n">is</span><span class="o">.</span><span class="n">item</span><span class="o">,</span> <span class="n">score</span><span class="o">)</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| |
| <span class="c1">// sum the standardized score if same item |
| </span> <span class="k">val</span> <span class="n">combined</span> <span class="k">=</span> <span class="n">standard</span><span class="o">.</span><span class="n">flatten</span> <span class="c1">// Array of ItemScore |
| </span> <span class="o">.</span><span class="n">groupBy</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">item</span><span class="o">)</span> <span class="c1">// groupBy item id |
| </span> <span class="o">.</span><span class="n">mapValues</span><span class="o">(</span><span class="n">itemScores</span> <span class="k">=></span> <span class="n">itemScores</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">score</span><span class="o">).</span><span class="n">reduce</span><span class="o">(</span><span class="k">_</span> <span class="o">+</span> <span class="k">_</span><span class="o">))</span> |
| <span class="o">.</span><span class="n">toArray</span> <span class="c1">// array of (item id, score) |
| </span> <span class="o">.</span><span class="n">sortBy</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">_2</span><span class="o">)(</span><span class="nc">Ordering</span><span class="o">.</span><span class="nc">Double</span><span class="o">.</span><span class="n">reverse</span><span class="o">)</span> |
| <span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="n">query</span><span class="o">.</span><span class="n">num</span><span class="o">)</span> |
| <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="k">case</span> <span class="o">(</span><span class="n">k</span><span class="o">,</span><span class="n">v</span><span class="o">)</span> <span class="k">=></span> <span class="nc">ItemScore</span><span class="o">(</span><span class="n">k</span><span class="o">,</span> <span class="n">v</span><span class="o">)</span> <span class="o">}</span> |
| |
| <span class="k">new</span> <span class="nc">PredictedResult</span><span class="o">(</span><span class="n">combined</span><span class="o">)</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| |
| </pre></td></tr></tbody></table> </div> <div class="alert-message note"><p>You may combine results of multiple algorithms in different ways based on your requirements.</p></div><h3 id='step-5.-modify-engine.scala-and-engine.json' class='header-anchors'>Step 5. Modify Engine.scala and engine.json</h3><p>Modify Engine.scala to include the new algorithm <code>LikeAlgorithm</code> class and give it the name <code>"likealgo"</code> as shown below. This defines an Engine with two available algorithms named as <code>"als"</code> and <code>"likealgo"</code>:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 |
| 2 |
| 3 |
| 4 |
| 5 |
| 6 |
| 7 |
| 8 |
| 9 |
| 10 |
| 11 |
| 12 |
| 13 |
| 14 |
| 15</pre></td><td class="code"><pre><span class="o">...</span> |
| |
| <span class="k">object</span> <span class="nc">SimilarProductEngine</span> <span class="k">extends</span> <span class="nc">IEngineFactory</span> <span class="o">{</span> |
| <span class="k">def</span> <span class="n">apply</span><span class="o">()</span> <span class="k">=</span> <span class="o">{</span> |
| <span class="k">new</span> <span class="nc">Engine</span><span class="o">(</span> |
| <span class="n">classOf</span><span class="o">[</span><span class="kt">DataSource</span><span class="o">],</span> |
| <span class="n">classOf</span><span class="o">[</span><span class="kt">Preparator</span><span class="o">],</span> |
| <span class="nc">Map</span><span class="o">(</span><span class="s">"als"</span> <span class="o">-></span> <span class="n">classOf</span><span class="o">[</span><span class="kt">ALSAlgorithm</span><span class="o">],</span> |
| <span class="s">"likealgo"</span> <span class="o">-></span> <span class="n">classOf</span><span class="o">[</span><span class="kt">LikeAlgorithm</span><span class="o">]),</span> <span class="c1">// ADDED |
| </span> <span class="n">classOf</span><span class="o">[</span><span class="kt">Serving</span><span class="o">])</span> |
| <span class="o">}</span> |
| <span class="o">}</span> |
| |
| <span class="o">...</span> |
| |
| </pre></td></tr></tbody></table> </div> <p>Next, in order to train and deploy two algorithms for this engine, we also need to modify engine.json to include the new algorithm. The <code>"algorithms"</code> parameter is an array of each algorithm's name and parameters. By default, it has the one algorithm <code>"als"</code> and its parameter. Add another JSON for the new algorithm named <code>"likealgo"</code> and its parameter to the <code>"algorithms"</code> array, as shown below:</p><div class="highlight json"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 |
| 2 |
| 3 |
| 4 |
| 5 |
| 6 |
| 7 |
| 8 |
| 9 |
| 10 |
| 11 |
| 12 |
| 13 |
| 14 |
| 15 |
| 16 |
| 17 |
| 18 |
| 19 |
| 20 |
| 21 |
| 22 |
| 23 |
| 24 |
| 25 |
| 26</pre></td><td class="code"><pre><span class="w"> |
| </span><span class="p">{</span><span class="w"> |
| </span><span class="err">...</span><span class="w"> |
| |
| </span><span class="s2">"algorithms"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"> |
| </span><span class="p">{</span><span class="w"> |
| </span><span class="s2">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"als"</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"params"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> |
| </span><span class="s2">"rank"</span><span class="p">:</span><span class="w"> </span><span class="mi">10</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"numIterations"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="mi">20</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"lambda"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.01</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"seed"</span><span class="p">:</span><span class="w"> </span><span class="mi">3</span><span class="w"> |
| </span><span class="p">}</span><span class="w"> |
| </span><span class="p">},</span><span class="w"> |
| </span><span class="p">{</span><span class="w"> |
| </span><span class="s2">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"likealgo"</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"params"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> |
| </span><span class="s2">"rank"</span><span class="p">:</span><span class="w"> </span><span class="mi">8</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"numIterations"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="mi">15</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"lambda"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.01</span><span class="p">,</span><span class="w"> |
| </span><span class="s2">"seed"</span><span class="p">:</span><span class="w"> </span><span class="mi">3</span><span class="w"> |
| </span><span class="p">}</span><span class="w"> |
| </span><span class="p">}</span><span class="w"> |
| </span><span class="p">]</span><span class="w"> |
| </span><span class="p">}</span><span class="w"> |
| |
| </span></pre></td></tr></tbody></table> </div> <div class="alert-message info"><p>You may notice that the parameters of the new <code>"likealgo"</code> contains the same fields as <code>"als"</code>. It is just becasuse the <code>LikeAlgorithm</code> class extends the original <code>ALSAlgorithm</code> class and shares the same algorithm parameter class defintion. If the other algorithm you add has its own parameter class, you just need to specify them inside its <code>params</code> field accordingly.</p></div><p>That's it! Now you have a engine configured with two algorithms.</p><h3 id='sample-data-with-"like"-and-"dislike"-events' class='header-anchors'>Sample data with "like" and "dislike" events</h3><p>For demonstration purpose, a sample import script is also provided for you to quickly test this engine. The script is modified from the original one used in <a href="/templates/similarproduct/quickstart/#sample-data-for-quick-start">Quick Start</a> with the addition of importing like and dislike events.</p><p>You could find the import script in <code>examples/scala-parallel-similarproduct-multi/data/import_eventserver.py</code>.</p><p>You can create a new App. Go to the directory <code>examples/scala-parallel-similarproduct-multi</code>. Execute the following to import the data (Replace the value of access_key parameter with your <strong>Access Key</strong>):</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$ </span>python data/import_eventserver.py --access_key 3mZWDzci2D5YsqAnqNnXH9SB6Rg3dsTBs8iHkK6X2i54IQsIZI1eEeQQyMfs7b3F |
| </pre></td></tr></tbody></table> </div> <div class="alert-message warning"><p>If you see error <strong>TypeError: <strong>init</strong>() got an unexpected keyword argument 'access_key'</strong>, please update the Python SDK to the latest version.</p></div><p>You are ready to run pio build, train and deploy as described in the <a href="/templates/similarproduct/quickstart/#deploy-the-engine-as-a-service">Quick Start</a>.</p></div></div></div></div><footer><div class="container"><div class="seperator"></div><div class="row"><div class="col-md-6 col-xs-6 footer-link-column"><div class="footer-link-column-row"><h4>Community</h4><ul><li><a href="//docs.prediction.io/install/" target="blank">Download</a></li><li><a href="//docs.prediction.io/" target="blank">Docs</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">GitHub</a></li><li><a href="mailto:user-subscribe@predictionio.incubator.apache.org" target="blank">Subscribe to User Mailing List</a></li><li><a href="//stackoverflow.com/questions/tagged/predictionio" target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6 col-xs-6 footer-link-column"><div class="footer-link-column-row"><h4>Contribute</h4><ul><li><a href="//predictionio.incubator.apache.org/community/contribute-code/" target="blank">Contribute</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">Source Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO" target="blank">Bug Tracker</a></li><li><a href="mailto:dev-subscribe@predictionio.incubator.apache.org" target="blank">Subscribe to Development Mailing List</a></li></ul></div></div></div></div><div id="footer-bottom"><div class="container"><div class="row"><div class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO" src="/images/logos/logo-white-d1e9c6e6.png"/></div><div id="social-icons-wrapper"><a class="github-button" href="https://github.com/apache/incubator-predictionio" data-style="mega" data-count-href="/apache/incubator-predictionio/stargazers" data-count-api="/repos/apache/incubator-predictionio#stargazers_count" data-count-aria-label="# stargazers on GitHub" aria-label="Star apache/incubator-predictionio on GitHub">Star</a> <a class="github-button" href="https://github.com/apache/incubator-predictionio/fork" data-icon="octicon-git-branch" data-style="mega" data-count-href="/apache/incubator-predictionio/network" data-count-api="/repos/apache/incubator-predictionio#forks_count" data-count-aria-label="# forks on GitHub" aria-label="Fork apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs" async="" defer="" src="https://buttons.github.io/buttons.js"></script><a href="//www.facebook.com/predictionio" target="blank"><img alt="PredictionIO on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a href="//twitter.com/predictionio" target="blank"><img alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a> </div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){ |
| (w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t); |
| e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e); |
| })(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st'); |
| |
| _st('install','HaUfpXXV87xoB_zzCQ45');</script><script src="/javascripts/application-5a24945b.js"></script></body></html> |
