Google Git
Sign in
apache / madlib-site / 388c4b34a08c96e059baca60f17350db327491ca / . / community-artifacts / Deep-learning / Preprocessor-for-images-v1.ipynb
blob: 2ab4b91b823887f7e4113179309be64b93e331fe [file] [log] [blame]
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Preprocessor for image data\n",
"This is a mini-batch preprocessor utility for image data:\n",
"* training_preprocessor_dl() for training datasets\n",
"* validation_preprocessor_dl() for validation datasets\n",
"\n",
"Note that there is a separate mini-batch preprocessor utility for general use cases\n",
"http://madlib.apache.org/docs/latest/group__grp__minibatch__preprocessing.html\n",
"\n",
"The preprocessor for image data was added in MADlib 1.16.\n",
"\n",
"## Table of contents\n",
"\n",
"<a href=\"#load_data\">1. Load data</a>\n",
"\n",
"<a href=\"#pp_train\">2. Run preprocessor for training image data</a>\n",
"\n",
"<a href=\"#pp_val\">3. Run preprocessor for validation image data</a>\n",
"\n",
"<a href=\"#load_data2\">4. Load data, another format</a>\n",
"\n",
"<a href=\"#pp_train2\">5. Run preprocessor for training image data</a>\n",
"\n",
"<a href=\"#pp_val2\">6. Run preprocessor for validation image data</a>\n",
"\n",
"<a href=\"#change_buffer\">7. Change buffer size</a>\n",
"\n",
"<a href=\"#set_num_classes\">8. Setting number of classes</a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13: ShimWarning: The `IPython.config` package has been deprecated since IPython 4.0. You should import from traitlets.config instead.\n",
" \"You should import from traitlets.config instead.\", ShimWarning)\n",
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5: UserWarning: IPython.utils.traitlets has moved to a top-level traitlets package.\n",
" warn(\"IPython.utils.traitlets has moved to a top-level traitlets package.\")\n"
]
}
],
"source": [
"%load_ext sql"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"u'Connected: gpadmin@madlib'"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Greenplum Database 5.x on GCP (PM demo machine)\n",
"%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
" \n",
"# PostgreSQL local\n",
"#%sql postgresql://fmcquillan@localhost:5432/madlib"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>version</th>\n",
" </tr>\n",
" <tr>\n",
" <td>MADlib version: 1.16-dev, git revision: rel/v1.15.1-98-g544a8e5, cmake configuration time: Mon May 20 16:40:50 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-98-g544a8e5, cmake configuration time: Mon May 20 16:40:50 UTC 2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5',)]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%sql select madlib.version();\n",
"#%sql select version();"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"load_data\"></a>\n",
"# 1. Load data\n",
"\n",
"Create an artificial 2x2 resolution color image data set with 3 possible classifications. The RGB values are per-pixel arrays:"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"52 rows affected.\n",
"52 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>rgb</th>\n",
" <th>species</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[76, 125, 240], [191, 13, 20]], [[153, 77, 7], [41, 143, 172]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[15, 126, 174], [246, 129, 81]], [[143, 220, 157], [96, 207, 223]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[84, 24, 1], [201, 28, 77]], [[70, 12, 11], [83, 33, 165]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[40, 206, 47], [211, 138, 62]], [[82, 56, 52], [210, 137, 195]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[38, 35, 125], [5, 18, 209]], [[29, 19, 153], [57, 95, 223]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[107, 50, 102], [15, 210, 142]], [[222, 1, 97], [103, 63, 179]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[115, 133, 40], [55, 51, 78]], [[89, 176, 83], [108, 129, 112]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[27, 169, 222], [249, 239, 73]], [[43, 85, 88], [253, 227, 54]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[68, 157, 61], [246, 60, 176]], [[123, 100, 230], [175, 178, 64]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[4, 172, 224], [116, 42, 251]], [[30, 8, 244], [12, 81, 31]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[169, 28, 68], [223, 26, 136]], [[124, 87, 126], [184, 7, 250]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[237, 168, 205], [45, 7, 210]], [[217, 231, 70], [3, 226, 100]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[214, 112, 91], [246, 209, 4]], [[18, 21, 227], [44, 157, 95]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[27, 22, 138], [21, 50, 119]], [[189, 255, 164], [196, 209, 125]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[23, 128, 141], [123, 138, 99]], [[236, 230, 88], [189, 234, 106]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[205, 151, 111], [44, 26, 139]], [[66, 163, 159], [116, 26, 92]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[191, 32, 68], [60, 203, 92]], [[188, 88, 215], [70, 186, 195]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[19, 128, 22], [125, 82, 227]], [[20, 193, 14], [45, 76, 80]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[235, 196, 235], [71, 55, 170]], [[103, 123, 230], [50, 215, 161]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[174, 231, 68], [112, 19, 87]], [[240, 41, 212], [66, 12, 232]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[26, 21, 79], [106, 230, 59]], [[46, 209, 130], [101, 123, 233]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[96, 27, 183], [1, 164, 100]], [[232, 232, 213], [251, 62, 197]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[18, 7, 47], [250, 10, 73]], [[15, 89, 180], [244, 148, 226]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[22, 71, 64], [255, 39, 160]], [[26, 222, 161], [190, 66, 137]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[23, 132, 228], [220, 168, 247]], [[226, 215, 241], [236, 32, 255]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[212, 244, 217], [182, 185, 239]], [[253, 249, 238], [36, 153, 7]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[202, 170, 34], [234, 24, 7]], [[99, 34, 11], [185, 160, 246]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[231, 138, 156], [250, 92, 165]], [[215, 8, 125], [201, 61, 208]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[6, 175, 244], [189, 184, 190]], [[103, 218, 167], [127, 225, 10]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[236, 195, 65], [226, 86, 41]], [[108, 242, 35], [200, 150, 250]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[19, 196, 13], [228, 219, 19]], [[147, 207, 208], [75, 141, 54]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[52, 181, 6], [63, 87, 243]], [[2, 152, 212], [88, 193, 64]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[227, 8, 224], [222, 216, 243]], [[161, 229, 215], [125, 248, 106]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[200, 181, 217], [254, 218, 13]], [[179, 224, 76], [10, 210, 78]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[167, 166, 99], [231, 239, 70]], [[239, 207, 36], [200, 194, 197]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[152, 66, 164], [2, 142, 108]], [[182, 102, 106], [144, 116, 29]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[191, 39, 66], [13, 202, 233]], [[179, 44, 209], [162, 114, 192]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[150, 136, 50], [91, 52, 202]], [[157, 217, 204], [43, 68, 130]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[174, 18, 5], [204, 130, 196]], [[243, 197, 210], [189, 174, 133]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[127, 38, 91], [63, 151, 242]], [[198, 201, 77], [250, 147, 234]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[95, 21, 24], [226, 167, 198]], [[244, 172, 146], [119, 113, 133]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[66, 66, 234], [199, 43, 105]], [[237, 134, 168], [132, 120, 110]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[197, 104, 225], [175, 59, 64]], [[197, 83, 34], [108, 25, 22]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[171, 141, 136], [48, 201, 203]], [[113, 179, 145], [156, 27, 127]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[195, 3, 155], [49, 80, 96]], [[153, 49, 15], [212, 113, 212]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[147, 63, 64], [169, 87, 235]], [[54, 223, 26], [254, 170, 139]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[59, 40, 204], [186, 74, 143]], [[189, 229, 192], [14, 69, 89]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[83, 45, 175], [39, 84, 66]], [[102, 149, 235], [189, 127, 32]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[153, 31, 71], [37, 207, 130]], [[76, 155, 61], [151, 42, 250]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[234, 7, 193], [67, 70, 20]], [[112, 245, 59], [196, 55, 161]]]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[34, 95, 216], [67, 252, 113]], [[97, 67, 150], [49, 197, 226]]]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[2, 121, 246], [252, 245, 224]], [[3, 182, 35], [73, 202, 147]]]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[[76, 125, 240], [191, 13, 20]], [[153, 77, 7], [41, 143, 172]]], u'bird'),\n",
" ([[[15, 126, 174], [246, 129, 81]], [[143, 220, 157], [96, 207, 223]]], u'bird'),\n",
" ([[[84, 24, 1], [201, 28, 77]], [[70, 12, 11], [83, 33, 165]]], u'dog'),\n",
" ([[[40, 206, 47], [211, 138, 62]], [[82, 56, 52], [210, 137, 195]]], u'bird'),\n",
" ([[[38, 35, 125], [5, 18, 209]], [[29, 19, 153], [57, 95, 223]]], u'cat'),\n",
" ([[[107, 50, 102], [15, 210, 142]], [[222, 1, 97], [103, 63, 179]]], u'dog'),\n",
" ([[[115, 133, 40], [55, 51, 78]], [[89, 176, 83], [108, 129, 112]]], u'dog'),\n",
" ([[[27, 169, 222], [249, 239, 73]], [[43, 85, 88], [253, 227, 54]]], u'bird'),\n",
" ([[[68, 157, 61], [246, 60, 176]], [[123, 100, 230], [175, 178, 64]]], u'dog'),\n",
" ([[[4, 172, 224], [116, 42, 251]], [[30, 8, 244], [12, 81, 31]]], u'dog'),\n",
" ([[[169, 28, 68], [223, 26, 136]], [[124, 87, 126], [184, 7, 250]]], u'cat'),\n",
" ([[[237, 168, 205], [45, 7, 210]], [[217, 231, 70], [3, 226, 100]]], u'cat'),\n",
" ([[[214, 112, 91], [246, 209, 4]], [[18, 21, 227], [44, 157, 95]]], u'dog'),\n",
" ([[[27, 22, 138], [21, 50, 119]], [[189, 255, 164], [196, 209, 125]]], u'bird'),\n",
" ([[[23, 128, 141], [123, 138, 99]], [[236, 230, 88], [189, 234, 106]]], u'bird'),\n",
" ([[[205, 151, 111], [44, 26, 139]], [[66, 163, 159], [116, 26, 92]]], u'dog'),\n",
" ([[[191, 32, 68], [60, 203, 92]], [[188, 88, 215], [70, 186, 195]]], u'cat'),\n",
" ([[[19, 128, 22], [125, 82, 227]], [[20, 193, 14], [45, 76, 80]]], u'bird'),\n",
" ([[[235, 196, 235], [71, 55, 170]], [[103, 123, 230], [50, 215, 161]]], u'dog'),\n",
" ([[[174, 231, 68], [112, 19, 87]], [[240, 41, 212], [66, 12, 232]]], u'cat'),\n",
" ([[[26, 21, 79], [106, 230, 59]], [[46, 209, 130], [101, 123, 233]]], u'bird'),\n",
" ([[[96, 27, 183], [1, 164, 100]], [[232, 232, 213], [251, 62, 197]]], u'cat'),\n",
" ([[[18, 7, 47], [250, 10, 73]], [[15, 89, 180], [244, 148, 226]]], u'bird'),\n",
" ([[[22, 71, 64], [255, 39, 160]], [[26, 222, 161], [190, 66, 137]]], u'dog'),\n",
" ([[[23, 132, 228], [220, 168, 247]], [[226, 215, 241], [236, 32, 255]]], u'cat'),\n",
" ([[[212, 244, 217], [182, 185, 239]], [[253, 249, 238], [36, 153, 7]]], u'cat'),\n",
" ([[[202, 170, 34], [234, 24, 7]], [[99, 34, 11], [185, 160, 246]]], u'dog'),\n",
" ([[[231, 138, 156], [250, 92, 165]], [[215, 8, 125], [201, 61, 208]]], u'cat'),\n",
" ([[[6, 175, 244], [189, 184, 190]], [[103, 218, 167], [127, 225, 10]]], u'dog'),\n",
" ([[[236, 195, 65], [226, 86, 41]], [[108, 242, 35], [200, 150, 250]]], u'bird'),\n",
" ([[[19, 196, 13], [228, 219, 19]], [[147, 207, 208], [75, 141, 54]]], u'cat'),\n",
" ([[[52, 181, 6], [63, 87, 243]], [[2, 152, 212], [88, 193, 64]]], u'cat'),\n",
" ([[[227, 8, 224], [222, 216, 243]], [[161, 229, 215], [125, 248, 106]]], u'cat'),\n",
" ([[[200, 181, 217], [254, 218, 13]], [[179, 224, 76], [10, 210, 78]]], u'dog'),\n",
" ([[[167, 166, 99], [231, 239, 70]], [[239, 207, 36], [200, 194, 197]]], u'bird'),\n",
" ([[[152, 66, 164], [2, 142, 108]], [[182, 102, 106], [144, 116, 29]]], u'dog'),\n",
" ([[[191, 39, 66], [13, 202, 233]], [[179, 44, 209], [162, 114, 192]]], u'dog'),\n",
" ([[[150, 136, 50], [91, 52, 202]], [[157, 217, 204], [43, 68, 130]]], u'dog'),\n",
" ([[[174, 18, 5], [204, 130, 196]], [[243, 197, 210], [189, 174, 133]]], u'bird'),\n",
" ([[[127, 38, 91], [63, 151, 242]], [[198, 201, 77], [250, 147, 234]]], u'bird'),\n",
" ([[[95, 21, 24], [226, 167, 198]], [[244, 172, 146], [119, 113, 133]]], u'cat'),\n",
" ([[[66, 66, 234], [199, 43, 105]], [[237, 134, 168], [132, 120, 110]]], u'cat'),\n",
" ([[[197, 104, 225], [175, 59, 64]], [[197, 83, 34], [108, 25, 22]]], u'cat'),\n",
" ([[[171, 141, 136], [48, 201, 203]], [[113, 179, 145], [156, 27, 127]]], u'cat'),\n",
" ([[[195, 3, 155], [49, 80, 96]], [[153, 49, 15], [212, 113, 212]]], u'cat'),\n",
" ([[[147, 63, 64], [169, 87, 235]], [[54, 223, 26], [254, 170, 139]]], u'bird'),\n",
" ([[[59, 40, 204], [186, 74, 143]], [[189, 229, 192], [14, 69, 89]]], u'cat'),\n",
" ([[[83, 45, 175], [39, 84, 66]], [[102, 149, 235], [189, 127, 32]]], u'dog'),\n",
" ([[[153, 31, 71], [37, 207, 130]], [[76, 155, 61], [151, 42, 250]]], u'dog'),\n",
" ([[[234, 7, 193], [67, 70, 20]], [[112, 245, 59], [196, 55, 161]]], u'dog'),\n",
" ([[[34, 95, 216], [67, 252, 113]], [[97, 67, 150], [49, 197, 226]]], u'bird'),\n",
" ([[[2, 121, 246], [252, 245, 224]], [[3, 182, 35], [73, 202, 147]]], u'cat')]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS image_data;\n",
"\n",
"CREATE TABLE image_data AS (\n",
" SELECT ARRAY[\n",
" ARRAY[\n",
" ARRAY[(random() * 256)::integer, -- pixel (1,1)\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer],\n",
" ARRAY[(random() * 256)::integer, -- pixel (2,1)\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer]\n",
" ],\n",
" ARRAY[\n",
" ARRAY[(random() * 256)::integer, -- pixel (1,2)\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer],\n",
" ARRAY[(random() * 256)::integer, -- pixel (2,1)\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer]\n",
" ]\n",
" ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species\n",
" FROM generate_series(1, 52)\n",
");\n",
"\n",
"SELECT * FROM image_data;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"pp_train\"></a>\n",
"# 2. Run preprocessor for training image data\n",
"\n",
"Run the preprocessor to generate the packed output table:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"1 rows affected.\n",
"2 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>independent_var</th>\n",
" <th>dependent_var</th>\n",
" <th>buffer_id</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.0862745, 0.278431, 0.25098], [1.0, 0.152941, 0.627451]], [[0.101961, 0.870588, 0.631373], [0.745098, 0.258824, 0.537255]]], [[[0.0588235, 0.494118, 0.682353], [0.964706, 0.505882, 0.317647]], [[0.560784, 0.862745, 0.615686], [0.376471, 0.811765, 0.87451]]], [[[0.156863, 0.807843, 0.184314], [0.827451, 0.541176, 0.243137]], [[0.321569, 0.219608, 0.203922], [0.823529, 0.537255, 0.764706]]], [[[0.419608, 0.196078, 0.4], [0.0588235, 0.823529, 0.556863]], [[0.870588, 0.00392157, 0.380392], [0.403922, 0.247059, 0.701961]]], [[[0.929412, 0.658824, 0.803922], [0.176471, 0.027451, 0.823529]], [[0.85098, 0.905882, 0.27451], [0.0117647, 0.886275, 0.392157]]], [[[0.00784314, 0.47451, 0.964706], [0.988235, 0.960784, 0.878431]], [[0.0117647, 0.713726, 0.137255], [0.286275, 0.792157, 0.576471]]], [[[0.92549, 0.764706, 0.254902], [0.886275, 0.337255, 0.160784]], [[0.423529, 0.94902, 0.137255], [0.784314, 0.588235, 0.980392]]], [[[0.376471, 0.105882, 0.717647], [0.00392157, 0.643137, 0.392157]], [[0.909804, 0.909804, 0.835294], [0.984314, 0.243137, 0.772549]]], [[[0.32549, 0.176471, 0.686275], [0.152941, 0.329412, 0.258824]], [[0.4, 0.584314, 0.921569], [0.741176, 0.498039, 0.12549]]], [[[0.498039, 0.14902, 0.356863], [0.247059, 0.592157, 0.94902]], [[0.776471, 0.788235, 0.301961], [0.980392, 0.576471, 0.917647]]], [[[0.105882, 0.0862745, 0.541176], [0.0823529, 0.196078, 0.466667]], [[0.741176, 1.0, 0.643137], [0.768628, 0.819608, 0.490196]]], [[[0.803922, 0.592157, 0.435294], [0.172549, 0.101961, 0.545098]], [[0.258824, 0.639216, 0.623529], [0.454902, 0.101961, 0.360784]]], [[[0.831373, 0.956863, 0.85098], [0.713726, 0.72549, 0.937255]], [[0.992157, 0.976471, 0.933333], [0.141176, 0.6, 0.027451]]], [[[0.905882, 0.541176, 0.611765], [0.980392, 0.360784, 0.647059]], [[0.843137, 0.0313726, 0.490196], [0.788235, 0.239216, 0.815686]]], [[[0.596078, 0.258824, 0.643137], [0.00784314, 0.556863, 0.423529]], [[0.713726, 0.4, 0.415686], [0.564706, 0.454902, 0.113725]]], [[[0.0156863, 0.67451, 0.878431], [0.454902, 0.164706, 0.984314]], [[0.117647, 0.0313726, 0.956863], [0.0470588, 0.317647, 0.121569]]], [[[0.682353, 0.905882, 0.266667], [0.439216, 0.0745098, 0.341176]], [[0.941177, 0.160784, 0.831373], [0.258824, 0.0470588, 0.909804]]], [[[0.203922, 0.709804, 0.0235294], [0.247059, 0.341176, 0.952941]], [[0.00784314, 0.596078, 0.831373], [0.345098, 0.756863, 0.25098]]], [[[0.917647, 0.027451, 0.756863], [0.262745, 0.27451, 0.0784314]], [[0.439216, 0.960784, 0.231373], [0.768628, 0.215686, 0.631373]]], [[[0.670588, 0.552941, 0.533333], [0.188235, 0.788235, 0.796079]], [[0.443137, 0.701961, 0.568627], [0.611765, 0.105882, 0.498039]]], [[[0.0745098, 0.501961, 0.0862745], [0.490196, 0.321569, 0.890196]], [[0.0784314, 0.756863, 0.054902], [0.176471, 0.298039, 0.313726]]], [[[0.588235, 0.533333, 0.196078], [0.356863, 0.203922, 0.792157]], [[0.615686, 0.85098, 0.8], [0.168627, 0.266667, 0.509804]]], [[[0.105882, 0.662745, 0.870588], [0.976471, 0.937255, 0.286275]], [[0.168627, 0.333333, 0.345098], [0.992157, 0.890196, 0.211765]]], [[[0.576471, 0.247059, 0.25098], [0.662745, 0.341176, 0.921569]], [[0.211765, 0.87451, 0.101961], [0.996078, 0.666667, 0.545098]]], [[[0.784314, 0.709804, 0.85098], [0.996078, 0.854902, 0.0509804]], [[0.701961, 0.878431, 0.298039], [0.0392157, 0.823529, 0.305882]]], [[[0.258824, 0.258824, 0.917647], [0.780392, 0.168627, 0.411765]], [[0.929412, 0.52549, 0.658824], [0.517647, 0.470588, 0.431373]]]]</td>\n",
" <td>[[0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.45098, 0.521569, 0.156863], [0.215686, 0.2, 0.305882]], [[0.34902, 0.690196, 0.32549], [0.423529, 0.505882, 0.439216]]], [[[0.0901961, 0.501961, 0.552941], [0.482353, 0.541176, 0.388235]], [[0.92549, 0.901961, 0.345098], [0.741176, 0.917647, 0.415686]]], [[[0.921569, 0.768628, 0.921569], [0.278431, 0.215686, 0.666667]], [[0.403922, 0.482353, 0.901961], [0.196078, 0.843137, 0.631373]]], [[[0.772549, 0.407843, 0.882353], [0.686275, 0.231373, 0.25098]], [[0.772549, 0.32549, 0.133333], [0.423529, 0.0980392, 0.0862745]]], [[[0.133333, 0.372549, 0.847059], [0.262745, 0.988235, 0.443137]], [[0.380392, 0.262745, 0.588235], [0.192157, 0.772549, 0.886275]]], [[[0.0901961, 0.517647, 0.894118], [0.862745, 0.658824, 0.968628]], [[0.886275, 0.843137, 0.945098], [0.92549, 0.12549, 1.0]]], [[[0.372549, 0.0823529, 0.0941177], [0.886275, 0.654902, 0.776471]], [[0.956863, 0.67451, 0.572549], [0.466667, 0.443137, 0.521569]]], [[[0.74902, 0.12549, 0.266667], [0.235294, 0.796079, 0.360784]], [[0.737255, 0.345098, 0.843137], [0.27451, 0.729412, 0.764706]]], [[[0.6, 0.121569, 0.278431], [0.145098, 0.811765, 0.509804]], [[0.298039, 0.607843, 0.239216], [0.592157, 0.164706, 0.980392]]], [[[0.764706, 0.0117647, 0.607843], [0.192157, 0.313726, 0.376471]], [[0.6, 0.192157, 0.0588235], [0.831373, 0.443137, 0.831373]]], [[[0.298039, 0.490196, 0.941177], [0.74902, 0.0509804, 0.0784314]], [[0.6, 0.301961, 0.027451], [0.160784, 0.560784, 0.67451]]], [[[0.792157, 0.666667, 0.133333], [0.917647, 0.0941177, 0.027451]], [[0.388235, 0.133333, 0.0431373], [0.72549, 0.627451, 0.964706]]], [[[0.890196, 0.0313726, 0.878431], [0.870588, 0.847059, 0.952941]], [[0.631373, 0.898039, 0.843137], [0.490196, 0.972549, 0.415686]]], [[[0.0705882, 0.027451, 0.184314], [0.980392, 0.0392157, 0.286275]], [[0.0588235, 0.34902, 0.705882], [0.956863, 0.580392, 0.886275]]], [[[0.266667, 0.615686, 0.239216], [0.964706, 0.235294, 0.690196]], [[0.482353, 0.392157, 0.901961], [0.686275, 0.698039, 0.25098]]], [[[0.839216, 0.439216, 0.356863], [0.964706, 0.819608, 0.0156863]], [[0.0705882, 0.0823529, 0.890196], [0.172549, 0.615686, 0.372549]]], [[[0.662745, 0.109804, 0.266667], [0.87451, 0.101961, 0.533333]], [[0.486275, 0.341176, 0.494118], [0.721569, 0.027451, 0.980392]]], [[[0.0745098, 0.768628, 0.0509804], [0.894118, 0.858824, 0.0745098]], [[0.576471, 0.811765, 0.815686], [0.294118, 0.552941, 0.211765]]], [[[0.654902, 0.65098, 0.388235], [0.905882, 0.937255, 0.27451]], [[0.937255, 0.811765, 0.141176], [0.784314, 0.760784, 0.772549]]], [[[0.329412, 0.0941177, 0.00392157], [0.788235, 0.109804, 0.301961]], [[0.27451, 0.0470588, 0.0431373], [0.32549, 0.129412, 0.647059]]], [[[0.682353, 0.0705882, 0.0196078], [0.8, 0.509804, 0.768628]], [[0.952941, 0.772549, 0.823529], [0.741176, 0.682353, 0.521569]]], [[[0.74902, 0.152941, 0.258824], [0.0509804, 0.792157, 0.913726]], [[0.701961, 0.172549, 0.819608], [0.635294, 0.447059, 0.752941]]], [[[0.101961, 0.0823529, 0.309804], [0.415686, 0.901961, 0.231373]], [[0.180392, 0.819608, 0.509804], [0.396078, 0.482353, 0.913726]]], [[[0.0235294, 0.686275, 0.956863], [0.741176, 0.721569, 0.745098]], [[0.403922, 0.854902, 0.654902], [0.498039, 0.882353, 0.0392157]]], [[[0.231373, 0.156863, 0.8], [0.729412, 0.290196, 0.560784]], [[0.741176, 0.898039, 0.752941], [0.054902, 0.270588, 0.34902]]], [[[0.14902, 0.137255, 0.490196], [0.0196078, 0.0705882, 0.819608]], [[0.113725, 0.0745098, 0.6], [0.223529, 0.372549, 0.87451]]]]</td>\n",
" <td>[[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0]]</td>\n",
" <td>1</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[[[0.0862745, 0.278431, 0.25098], [1.0, 0.152941, 0.627451]], [[0.101961, 0.870588, 0.631373], [0.745098, 0.258824, 0.537255]]], [[[0.0588235, 0.494118, 0.682353], [0.964706, 0.505882, 0.317647]], [[0.560784, 0.862745, 0.615686], [0.376471, 0.811765, 0.87451]]], [[[0.156863, 0.807843, 0.184314], [0.827451, 0.541176, 0.243137]], [[0.321569, 0.219608, 0.203922], [0.823529, 0.537255, 0.764706]]], [[[0.419608, 0.196078, 0.4], [0.0588235, 0.823529, 0.556863]], [[0.870588, 0.00392157, 0.380392], [0.403922, 0.247059, 0.701961]]], [[[0.929412, 0.658824, 0.803922], [0.176471, 0.027451, 0.823529]], [[0.85098, 0.905882, 0.27451], [0.0117647, 0.886275, 0.392157]]], [[[0.00784314, 0.47451, 0.964706], [0.988235, 0.960784, 0.878431]], [[0.0117647, 0.713726, 0.137255], [0.286275, 0.792157, 0.576471]]], [[[0.92549, 0.764706, 0.254902], [0.886275, 0.337255, 0.160784]], [[0.423529, 0.94902, 0.137255], [0.784314, 0.588235, 0.980392]]], [[[0.376471, 0.105882, 0.717647], [0.00392157, 0.643137, 0.392157]], [[0.909804, 0.909804, 0.835294], [0.984314, 0.243137, 0.772549]]], [[[0.32549, 0.176471, 0.686275], [0.152941, 0.329412, 0.258824]], [[0.4, 0.584314, 0.921569], [0.741176, 0.498039, 0.12549]]], [[[0.498039, 0.14902, 0.356863], [0.247059, 0.592157, 0.94902]], [[0.776471, 0.788235, 0.301961], [0.980392, 0.576471, 0.917647]]], [[[0.105882, 0.0862745, 0.541176], [0.0823529, 0.196078, 0.466667]], [[0.741176, 1.0, 0.643137], [0.768628, 0.819608, 0.490196]]], [[[0.803922, 0.592157, 0.435294], [0.172549, 0.101961, 0.545098]], [[0.258824, 0.639216, 0.623529], [0.454902, 0.101961, 0.360784]]], [[[0.831373, 0.956863, 0.85098], [0.713726, 0.72549, 0.937255]], [[0.992157, 0.976471, 0.933333], [0.141176, 0.6, 0.027451]]], [[[0.905882, 0.541176, 0.611765], [0.980392, 0.360784, 0.647059]], [[0.843137, 0.0313726, 0.490196], [0.788235, 0.239216, 0.815686]]], [[[0.596078, 0.258824, 0.643137], [0.00784314, 0.556863, 0.423529]], [[0.713726, 0.4, 0.415686], [0.564706, 0.454902, 0.113725]]], [[[0.0156863, 0.67451, 0.878431], [0.454902, 0.164706, 0.984314]], [[0.117647, 0.0313726, 0.956863], [0.0470588, 0.317647, 0.121569]]], [[[0.682353, 0.905882, 0.266667], [0.439216, 0.0745098, 0.341176]], [[0.941177, 0.160784, 0.831373], [0.258824, 0.0470588, 0.909804]]], [[[0.203922, 0.709804, 0.0235294], [0.247059, 0.341176, 0.952941]], [[0.00784314, 0.596078, 0.831373], [0.345098, 0.756863, 0.25098]]], [[[0.917647, 0.027451, 0.756863], [0.262745, 0.27451, 0.0784314]], [[0.439216, 0.960784, 0.231373], [0.768628, 0.215686, 0.631373]]], [[[0.670588, 0.552941, 0.533333], [0.188235, 0.788235, 0.796079]], [[0.443137, 0.701961, 0.568627], [0.611765, 0.105882, 0.498039]]], [[[0.0745098, 0.501961, 0.0862745], [0.490196, 0.321569, 0.890196]], [[0.0784314, 0.756863, 0.054902], [0.176471, 0.298039, 0.313726]]], [[[0.588235, 0.533333, 0.196078], [0.356863, 0.203922, 0.792157]], [[0.615686, 0.85098, 0.8], [0.168627, 0.266667, 0.509804]]], [[[0.105882, 0.662745, 0.870588], [0.976471, 0.937255, 0.286275]], [[0.168627, 0.333333, 0.345098], [0.992157, 0.890196, 0.211765]]], [[[0.576471, 0.247059, 0.25098], [0.662745, 0.341176, 0.921569]], [[0.211765, 0.87451, 0.101961], [0.996078, 0.666667, 0.545098]]], [[[0.784314, 0.709804, 0.85098], [0.996078, 0.854902, 0.0509804]], [[0.701961, 0.878431, 0.298039], [0.0392157, 0.823529, 0.305882]]], [[[0.258824, 0.258824, 0.917647], [0.780392, 0.168627, 0.411765]], [[0.929412, 0.52549, 0.658824], [0.517647, 0.470588, 0.431373]]]], [[0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]], 0),\n",
" ([[[[0.45098, 0.521569, 0.156863], [0.215686, 0.2, 0.305882]], [[0.34902, 0.690196, 0.32549], [0.423529, 0.505882, 0.439216]]], [[[0.0901961, 0.501961, 0.552941], [0.482353, 0.541176, 0.388235]], [[0.92549, 0.901961, 0.345098], [0.741176, 0.917647, 0.415686]]], [[[0.921569, 0.768628, 0.921569], [0.278431, 0.215686, 0.666667]], [[0.403922, 0.482353, 0.901961], [0.196078, 0.843137, 0.631373]]], [[[0.772549, 0.407843, 0.882353], [0.686275, 0.231373, 0.25098]], [[0.772549, 0.32549, 0.133333], [0.423529, 0.0980392, 0.0862745]]], [[[0.133333, 0.372549, 0.847059], [0.262745, 0.988235, 0.443137]], [[0.380392, 0.262745, 0.588235], [0.192157, 0.772549, 0.886275]]], [[[0.0901961, 0.517647, 0.894118], [0.862745, 0.658824, 0.968628]], [[0.886275, 0.843137, 0.945098], [0.92549, 0.12549, 1.0]]], [[[0.372549, 0.0823529, 0.0941177], [0.886275, 0.654902, 0.776471]], [[0.956863, 0.67451, 0.572549], [0.466667, 0.443137, 0.521569]]], [[[0.74902, 0.12549, 0.266667], [0.235294, 0.796079, 0.360784]], [[0.737255, 0.345098, 0.843137], [0.27451, 0.729412, 0.764706]]], [[[0.6, 0.121569, 0.278431], [0.145098, 0.811765, 0.509804]], [[0.298039, 0.607843, 0.239216], [0.592157, 0.164706, 0.980392]]], [[[0.764706, 0.0117647, 0.607843], [0.192157, 0.313726, 0.376471]], [[0.6, 0.192157, 0.0588235], [0.831373, 0.443137, 0.831373]]], [[[0.298039, 0.490196, 0.941177], [0.74902, 0.0509804, 0.0784314]], [[0.6, 0.301961, 0.027451], [0.160784, 0.560784, 0.67451]]], [[[0.792157, 0.666667, 0.133333], [0.917647, 0.0941177, 0.027451]], [[0.388235, 0.133333, 0.0431373], [0.72549, 0.627451, 0.964706]]], [[[0.890196, 0.0313726, 0.878431], [0.870588, 0.847059, 0.952941]], [[0.631373, 0.898039, 0.843137], [0.490196, 0.972549, 0.415686]]], [[[0.0705882, 0.027451, 0.184314], [0.980392, 0.0392157, 0.286275]], [[0.0588235, 0.34902, 0.705882], [0.956863, 0.580392, 0.886275]]], [[[0.266667, 0.615686, 0.239216], [0.964706, 0.235294, 0.690196]], [[0.482353, 0.392157, 0.901961], [0.686275, 0.698039, 0.25098]]], [[[0.839216, 0.439216, 0.356863], [0.964706, 0.819608, 0.0156863]], [[0.0705882, 0.0823529, 0.890196], [0.172549, 0.615686, 0.372549]]], [[[0.662745, 0.109804, 0.266667], [0.87451, 0.101961, 0.533333]], [[0.486275, 0.341176, 0.494118], [0.721569, 0.027451, 0.980392]]], [[[0.0745098, 0.768628, 0.0509804], [0.894118, 0.858824, 0.0745098]], [[0.576471, 0.811765, 0.815686], [0.294118, 0.552941, 0.211765]]], [[[0.654902, 0.65098, 0.388235], [0.905882, 0.937255, 0.27451]], [[0.937255, 0.811765, 0.141176], [0.784314, 0.760784, 0.772549]]], [[[0.329412, 0.0941177, 0.00392157], [0.788235, 0.109804, 0.301961]], [[0.27451, 0.0470588, 0.0431373], [0.32549, 0.129412, 0.647059]]], [[[0.682353, 0.0705882, 0.0196078], [0.8, 0.509804, 0.768628]], [[0.952941, 0.772549, 0.823529], [0.741176, 0.682353, 0.521569]]], [[[0.74902, 0.152941, 0.258824], [0.0509804, 0.792157, 0.913726]], [[0.701961, 0.172549, 0.819608], [0.635294, 0.447059, 0.752941]]], [[[0.101961, 0.0823529, 0.309804], [0.415686, 0.901961, 0.231373]], [[0.180392, 0.819608, 0.509804], [0.396078, 0.482353, 0.913726]]], [[[0.0235294, 0.686275, 0.956863], [0.741176, 0.721569, 0.745098]], [[0.403922, 0.854902, 0.654902], [0.498039, 0.882353, 0.0392157]]], [[[0.231373, 0.156863, 0.8], [0.729412, 0.290196, 0.560784]], [[0.741176, 0.898039, 0.752941], [0.054902, 0.270588, 0.34902]]], [[[0.14902, 0.137255, 0.490196], [0.0196078, 0.0705882, 0.819608]], [[0.113725, 0.0745098, 0.6], [0.223529, 0.372549, 0.87451]]]], [[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0]], 1)]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
"\n",
"SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
" 'image_data_packed', -- Output table\n",
" 'species', -- Dependent variable\n",
" 'rgb', -- Independent variable\n",
" NULL, -- Buffer size\n",
" 255 -- Normalizing constant\n",
" );\n",
"\n",
"SELECT * FROM image_data_packed ORDER BY buffer_id;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For small datasets like in this example, buffer size is mainly determined by the number of segments in the database. For a Greenplum database with 2 segments, there will be 2 rows with a buffer size of 26. For PostgresSQL, there would be only one row with a buffer size of 52 since it is a single node database. For larger data sets, other factors go into computing buffers size besides number of segments. \n",
"\n",
"Review the output summary table:"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>source_table</th>\n",
" <th>output_table</th>\n",
" <th>dependent_varname</th>\n",
" <th>independent_varname</th>\n",
" <th>dependent_vartype</th>\n",
" <th>class_values</th>\n",
" <th>buffer_size</th>\n",
" <th>normalizing_const</th>\n",
" <th>num_classes</th>\n",
" </tr>\n",
" <tr>\n",
" <td>image_data</td>\n",
" <td>image_data_packed</td>\n",
" <td>species</td>\n",
" <td>rgb</td>\n",
" <td>text</td>\n",
" <td>[u'bird', u'cat', u'dog']</td>\n",
" <td>26</td>\n",
" <td>255.0</td>\n",
" <td>3</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 26, Decimal('255.0'), 3)]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"SELECT * FROM image_data_packed_summary;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"pp_val\"></a>\n",
"# 3. Run preprocessor for validation image data\n",
"\n",
"Run the preprocessor for the validation dataset. In this example, we use the same images for validation to demonstrate, but normally validation data is different than training data:"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"1 rows affected.\n",
"26 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>independent_var</th>\n",
" <th>dependent_var</th>\n",
" <th>buffer_id</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.290196, 0.105882, 0.152941], [0.47451, 0.670588, 0.356863]], [[0.101961, 0.458824, 0.188235], [0.584314, 0.921569, 0.854902]]], [[[0.941177, 0.92549, 0.34902], [0.137255, 0.360784, 0.411765]], [[0.0627451, 0.917647, 0.898039], [0.203922, 0.313726, 0.247059]]]]</td>\n",
" <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.815686, 0.294118, 0.568627], [0.603922, 0.662745, 0.870588]], [[0.913726, 0.352941, 0.0745098], [0.0117647, 0.596078, 0.6]]], [[[0.0509804, 0.262745, 0.933333], [0.894118, 0.603922, 0.0901961]], [[0.643137, 0.12549, 0.623529], [0.0392157, 0.713726, 0.819608]]]]</td>\n",
" <td>[[0, 1, 0], [0, 0, 1]]</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.14902, 0.52549, 0.6], [0.784314, 0.619608, 0.823529]], [[0.0823529, 0.0862745, 0.454902], [0.835294, 0.231373, 0.996078]]], [[[0.713726, 0.803922, 0.0156863], [0.678431, 0.415686, 0.470588]], [[0.156863, 0.85098, 0.941177], [0.27451, 0.141176, 0.72549]]]]</td>\n",
" <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.584314, 0.890196, 0.94902], [0.419608, 0.709804, 0.760784]], [[0.713726, 0.270588, 0.360784], [0.372549, 0.141176, 0.270588]]], [[[0.819608, 0.823529, 0.0980392], [0.262745, 0.713726, 0.552941]], [[0.203922, 0.890196, 0.945098], [0.156863, 0.235294, 0.466667]]]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.215686, 0.282353, 0.321569], [0.815686, 0.588235, 0.466667]], [[0.337255, 0.184314, 0.247059], [0.956863, 0.00392157, 0.329412]]], [[[0.129412, 0.388235, 0.270588], [0.980392, 0.623529, 0.984314]], [[0.780392, 0.639216, 0.658824], [0.192157, 0.105882, 0.815686]]]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.458824, 0.160784, 0.270588], [0.45098, 0.94902, 0.854902]], [[0.337255, 0.894118, 0.27451], [0.0431373, 0.65098, 0.988235]]], [[[0.0431373, 0.0862745, 0.180392], [0.772549, 0.615686, 1.0]], [[0.588235, 0.713726, 0.254902], [0.298039, 0.262745, 0.458824]]]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
" <td>5</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.00784314, 0.356863, 0.454902], [0.282353, 0.0784314, 0.670588]], [[0.564706, 0.4, 0.478431], [0.14902, 0.866667, 0.815686]]], [[[0.207843, 0.615686, 0.419608], [0.670588, 0.760784, 0.54902]], [[0.054902, 0.0313726, 0.52549], [0.678431, 0.0117647, 0.298039]]]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
" <td>6</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.666667, 0.490196, 0.419608], [0.478431, 0.529412, 0.462745]], [[0.564706, 0.709804, 0.231373], [0.176471, 0.701961, 0.819608]]], [[[0.113725, 0.764706, 0.337255], [0.439216, 0.803922, 0.796079]], [[0.6, 0.0745098, 0.243137], [0.54902, 0.929412, 0.580392]]]]</td>\n",
" <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
" <td>7</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.196078, 0.619608, 0.0862745], [0.180392, 0.933333, 0.0941177]], [[0.537255, 0.384314, 0.376471], [0.619608, 0.0509804, 0.941177]]], [[[0.960784, 0.113725, 0.14902], [0.415686, 0.301961, 0.356863]], [[0.027451, 0.721569, 0.0235294], [0.788235, 0.266667, 0.0784314]]]]</td>\n",
" <td>[[0, 1, 0], [1, 0, 0]]</td>\n",
" <td>8</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.529412, 0.0862745, 0.882353], [0.341176, 0.415686, 0.996078]], [[0.101961, 0.752941, 0.431373], [0.909804, 0.545098, 0.027451]]], [[[0.792157, 0.760784, 0.827451], [0.0862745, 0.0705882, 0.490196]], [[0.576471, 0.490196, 0.972549], [0.101961, 0.952941, 0.533333]]]]</td>\n",
" <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
" <td>9</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.180392, 0.709804, 0.509804], [1.0, 0.592157, 0.466667]], [[0.113725, 0.741176, 0.882353], [0.415686, 0.0941177, 0.905882]]], [[[0.784314, 0.576471, 0.905882], [0.360784, 0.0117647, 0.0980392]], [[0.980392, 0.0980392, 0.282353], [0.913726, 0.196078, 0.819608]]]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
" <td>10</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.572549, 0.431373, 0.345098], [0.509804, 0.447059, 0.87451]], [[0.592157, 0.32549, 0.211765], [0.00784314, 0.313726, 0.313726]]], [[[0.117647, 0.694118, 0.4], [0.196078, 0.505882, 0.188235]], [[0.956863, 0.329412, 0.27451], [0.0235294, 0.823529, 0.854902]]]]</td>\n",
" <td>[[1, 0, 0], [0, 0, 1]]</td>\n",
" <td>11</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.792157, 0.956863, 0.458824], [0.317647, 0.764706, 0.639216]], [[0.0235294, 0.270588, 0.635294], [0.615686, 0.737255, 0.74902]]], [[[0.745098, 0.219608, 0.301961], [0.776471, 0.196078, 0.0823529]], [[0.34902, 0.0980392, 0.443137], [0.360784, 0.196078, 0.419608]]]]</td>\n",
" <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
" <td>12</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.478431, 0.329412, 0.654902], [0.290196, 0.623529, 0.223529]], [[0.721569, 0.964706, 0.729412], [0.164706, 0.835294, 0.321569]]], [[[0.615686, 0.156863, 0.447059], [0.521569, 0.290196, 0.564706]], [[0.207843, 0.690196, 0.760784], [0.717647, 0.878431, 0.713726]]]]</td>\n",
" <td>[[0, 0, 1], [0, 1, 0]]</td>\n",
" <td>13</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.0431373, 0.32549, 0.803922], [0.356863, 0.0823529, 0.54902]], [[0.572549, 0.384314, 0.321569], [0.768628, 0.466667, 0.670588]]], [[[0.14902, 0.737255, 0.866667], [0.0, 0.243137, 0.65098]], [[0.956863, 0.705882, 0.972549], [0.721569, 0.341176, 0.996078]]]]</td>\n",
" <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
" <td>14</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.976471, 0.607843, 0.72549], [0.721569, 0.960784, 0.341176]], [[0.878431, 0.4, 0.858824], [0.164706, 0.964706, 0.0666667]]], [[[0.909804, 0.027451, 0.0588235], [0.32549, 0.486275, 0.537255]], [[0.658824, 0.137255, 0.827451], [0.27451, 0.360784, 0.545098]]]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
" <td>15</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.0862745, 0.709804, 0.0705882], [0.403922, 0.196078, 0.109804]], [[0.733333, 1.0, 0.466667], [0.815686, 0.541176, 0.0352941]]], [[[0.721569, 0.780392, 0.729412], [0.431373, 0.823529, 0.882353]], [[0.164706, 0.686275, 0.882353], [0.407843, 0.333333, 0.835294]]]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
" <td>16</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.862745, 0.803922, 0.662745], [0.529412, 0.666667, 0.568627]], [[0.552941, 0.72549, 0.894118], [0.0352941, 0.254902, 0.54902]]], [[[0.301961, 0.552941, 0.447059], [0.294118, 0.541176, 0.419608]], [[0.898039, 0.266667, 0.137255], [0.854902, 0.603922, 0.0117647]]]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0]]</td>\n",
" <td>17</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.0784314, 0.823529, 0.533333], [0.623529, 0.0588235, 0.619608]], [[0.329412, 0.12549, 0.0196078], [0.52549, 0.235294, 0.752941]]], [[[0.462745, 0.180392, 0.211765], [0.52549, 0.0313726, 0.933333]], [[0.305882, 0.760784, 0.360784], [0.12549, 0.639216, 0.52549]]]]</td>\n",
" <td>[[0, 0, 1], [1, 0, 0]]</td>\n",
" <td>18</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.701961, 0.564706, 0.0588235], [0.737255, 0.760784, 0.921569]], [[0.537255, 0.415686, 0.447059], [0.2, 0.984314, 1.0]]], [[[0.517647, 0.933333, 0.141176], [0.352941, 0.0352941, 0.447059]], [[0.905882, 0.486275, 0.737255], [0.443137, 0.905882, 0.631373]]]]</td>\n",
" <td>[[1, 0, 0], [1, 0, 0]]</td>\n",
" <td>19</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.878431, 0.0352941, 0.176471], [0.419608, 0.207843, 0.258824]], [[0.243137, 0.741176, 0.882353], [0.298039, 0.356863, 0.207843]]], [[[0.0392157, 0.482353, 0.309804], [0.0509804, 0.737255, 0.768628]], [[0.231373, 0.94902, 0.290196], [0.262745, 0.878431, 0.596078]]]]</td>\n",
" <td>[[0, 0, 1], [0, 1, 0]]</td>\n",
" <td>20</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.376471, 0.733333, 0.662745], [0.121569, 0.25098, 0.360784]], [[0.686275, 0.309804, 0.0941177], [0.443137, 0.231373, 0.631373]]], [[[0.239216, 0.721569, 0.658824], [0.764706, 0.529412, 0.172549]], [[0.694118, 0.670588, 0.52549], [0.729412, 0.113725, 0.427451]]]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
" <td>21</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.85098, 0.866667, 0.113725], [0.478431, 0.572549, 0.152941]], [[0.964706, 0.878431, 0.203922], [0.694118, 0.647059, 0.431373]]], [[[0.67451, 0.831373, 0.839216], [0.67451, 0.752941, 0.713726]], [[0.705882, 0.933333, 0.129412], [0.917647, 0.184314, 0.372549]]]]</td>\n",
" <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
" <td>22</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.0627451, 0.670588, 0.00392157], [0.270588, 0.0941177, 0.380392]], [[0.0, 0.752941, 0.505882], [0.25098, 0.109804, 0.188235]]], [[[0.937255, 0.694118, 0.513726], [0.529412, 0.713726, 0.752941]], [[0.247059, 0.368627, 0.513726], [0.776471, 0.541176, 0.2]]]]</td>\n",
" <td>[[0, 0, 1], [0, 0, 1]]</td>\n",
" <td>23</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.203922, 0.627451, 0.792157], [0.835294, 0.482353, 0.462745]], [[0.662745, 0.321569, 0.133333], [0.411765, 0.027451, 0.839216]]], [[[0.0627451, 0.929412, 0.552941], [0.490196, 0.137255, 0.4]], [[0.352941, 0.25098, 0.882353], [0.92549, 0.403922, 0.839216]]]]</td>\n",
" <td>[[0, 0, 1], [1, 0, 0]]</td>\n",
" <td>24</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[[[0.603922, 0.529412, 0.443137], [0.0352941, 0.164706, 0.376471]], [[0.729412, 0.678431, 0.905882], [0.439216, 0.427451, 0.14902]]], [[[0.160784, 0.752941, 0.52549], [0.533333, 0.403922, 0.588235]], [[0.2, 0.407843, 0.858824], [0.290196, 0.788235, 0.858824]]]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0]]</td>\n",
" <td>25</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[[[0.290196, 0.105882, 0.152941], [0.47451, 0.670588, 0.356863]], [[0.101961, 0.458824, 0.188235], [0.584314, 0.921569, 0.854902]]], [[[0.941177, 0.92549, 0.34902], [0.137255, 0.360784, 0.411765]], [[0.0627451, 0.917647, 0.898039], [0.203922, 0.313726, 0.247059]]]], [[1, 0, 0], [0, 0, 1]], 0),\n",
" ([[[[0.815686, 0.294118, 0.568627], [0.603922, 0.662745, 0.870588]], [[0.913726, 0.352941, 0.0745098], [0.0117647, 0.596078, 0.6]]], [[[0.0509804, 0.262745, 0.933333], [0.894118, 0.603922, 0.0901961]], [[0.643137, 0.12549, 0.623529], [0.0392157, 0.713726, 0.819608]]]], [[0, 1, 0], [0, 0, 1]], 1),\n",
" ([[[[0.14902, 0.52549, 0.6], [0.784314, 0.619608, 0.823529]], [[0.0823529, 0.0862745, 0.454902], [0.835294, 0.231373, 0.996078]]], [[[0.713726, 0.803922, 0.0156863], [0.678431, 0.415686, 0.470588]], [[0.156863, 0.85098, 0.941177], [0.27451, 0.141176, 0.72549]]]], [[1, 0, 0], [0, 0, 1]], 2),\n",
" ([[[[0.584314, 0.890196, 0.94902], [0.419608, 0.709804, 0.760784]], [[0.713726, 0.270588, 0.360784], [0.372549, 0.141176, 0.270588]]], [[[0.819608, 0.823529, 0.0980392], [0.262745, 0.713726, 0.552941]], [[0.203922, 0.890196, 0.945098], [0.156863, 0.235294, 0.466667]]]], [[1, 0, 0], [0, 1, 0]], 3),\n",
" ([[[[0.215686, 0.282353, 0.321569], [0.815686, 0.588235, 0.466667]], [[0.337255, 0.184314, 0.247059], [0.956863, 0.00392157, 0.329412]]], [[[0.129412, 0.388235, 0.270588], [0.980392, 0.623529, 0.984314]], [[0.780392, 0.639216, 0.658824], [0.192157, 0.105882, 0.815686]]]], [[0, 1, 0], [0, 1, 0]], 4),\n",
" ([[[[0.458824, 0.160784, 0.270588], [0.45098, 0.94902, 0.854902]], [[0.337255, 0.894118, 0.27451], [0.0431373, 0.65098, 0.988235]]], [[[0.0431373, 0.0862745, 0.180392], [0.772549, 0.615686, 1.0]], [[0.588235, 0.713726, 0.254902], [0.298039, 0.262745, 0.458824]]]], [[0, 1, 0], [0, 1, 0]], 5),\n",
" ([[[[0.00784314, 0.356863, 0.454902], [0.282353, 0.0784314, 0.670588]], [[0.564706, 0.4, 0.478431], [0.14902, 0.866667, 0.815686]]], [[[0.207843, 0.615686, 0.419608], [0.670588, 0.760784, 0.54902]], [[0.054902, 0.0313726, 0.52549], [0.678431, 0.0117647, 0.298039]]]], [[0, 1, 0], [0, 1, 0]], 6),\n",
" ([[[[0.666667, 0.490196, 0.419608], [0.478431, 0.529412, 0.462745]], [[0.564706, 0.709804, 0.231373], [0.176471, 0.701961, 0.819608]]], [[[0.113725, 0.764706, 0.337255], [0.439216, 0.803922, 0.796079]], [[0.6, 0.0745098, 0.243137], [0.54902, 0.929412, 0.580392]]]], [[1, 0, 0], [0, 0, 1]], 7),\n",
" ([[[[0.196078, 0.619608, 0.0862745], [0.180392, 0.933333, 0.0941177]], [[0.537255, 0.384314, 0.376471], [0.619608, 0.0509804, 0.941177]]], [[[0.960784, 0.113725, 0.14902], [0.415686, 0.301961, 0.356863]], [[0.027451, 0.721569, 0.0235294], [0.788235, 0.266667, 0.0784314]]]], [[0, 1, 0], [1, 0, 0]], 8),\n",
" ([[[[0.529412, 0.0862745, 0.882353], [0.341176, 0.415686, 0.996078]], [[0.101961, 0.752941, 0.431373], [0.909804, 0.545098, 0.027451]]], [[[0.792157, 0.760784, 0.827451], [0.0862745, 0.0705882, 0.490196]], [[0.576471, 0.490196, 0.972549], [0.101961, 0.952941, 0.533333]]]], [[1, 0, 0], [1, 0, 0]], 9),\n",
" ([[[[0.180392, 0.709804, 0.509804], [1.0, 0.592157, 0.466667]], [[0.113725, 0.741176, 0.882353], [0.415686, 0.0941177, 0.905882]]], [[[0.784314, 0.576471, 0.905882], [0.360784, 0.0117647, 0.0980392]], [[0.980392, 0.0980392, 0.282353], [0.913726, 0.196078, 0.819608]]]], [[0, 1, 0], [0, 1, 0]], 10),\n",
" ([[[[0.572549, 0.431373, 0.345098], [0.509804, 0.447059, 0.87451]], [[0.592157, 0.32549, 0.211765], [0.00784314, 0.313726, 0.313726]]], [[[0.117647, 0.694118, 0.4], [0.196078, 0.505882, 0.188235]], [[0.956863, 0.329412, 0.27451], [0.0235294, 0.823529, 0.854902]]]], [[1, 0, 0], [0, 0, 1]], 11),\n",
" ([[[[0.792157, 0.956863, 0.458824], [0.317647, 0.764706, 0.639216]], [[0.0235294, 0.270588, 0.635294], [0.615686, 0.737255, 0.74902]]], [[[0.745098, 0.219608, 0.301961], [0.776471, 0.196078, 0.0823529]], [[0.34902, 0.0980392, 0.443137], [0.360784, 0.196078, 0.419608]]]], [[0, 0, 1], [0, 0, 1]], 12),\n",
" ([[[[0.478431, 0.329412, 0.654902], [0.290196, 0.623529, 0.223529]], [[0.721569, 0.964706, 0.729412], [0.164706, 0.835294, 0.321569]]], [[[0.615686, 0.156863, 0.447059], [0.521569, 0.290196, 0.564706]], [[0.207843, 0.690196, 0.760784], [0.717647, 0.878431, 0.713726]]]], [[0, 0, 1], [0, 1, 0]], 13),\n",
" ([[[[0.0431373, 0.32549, 0.803922], [0.356863, 0.0823529, 0.54902]], [[0.572549, 0.384314, 0.321569], [0.768628, 0.466667, 0.670588]]], [[[0.14902, 0.737255, 0.866667], [0.0, 0.243137, 0.65098]], [[0.956863, 0.705882, 0.972549], [0.721569, 0.341176, 0.996078]]]], [[1, 0, 0], [1, 0, 0]], 14),\n",
" ([[[[0.976471, 0.607843, 0.72549], [0.721569, 0.960784, 0.341176]], [[0.878431, 0.4, 0.858824], [0.164706, 0.964706, 0.0666667]]], [[[0.909804, 0.027451, 0.0588235], [0.32549, 0.486275, 0.537255]], [[0.658824, 0.137255, 0.827451], [0.27451, 0.360784, 0.545098]]]], [[1, 0, 0], [0, 1, 0]], 15),\n",
" ([[[[0.0862745, 0.709804, 0.0705882], [0.403922, 0.196078, 0.109804]], [[0.733333, 1.0, 0.466667], [0.815686, 0.541176, 0.0352941]]], [[[0.721569, 0.780392, 0.729412], [0.431373, 0.823529, 0.882353]], [[0.164706, 0.686275, 0.882353], [0.407843, 0.333333, 0.835294]]]], [[0, 1, 0], [0, 1, 0]], 16),\n",
" ([[[[0.862745, 0.803922, 0.662745], [0.529412, 0.666667, 0.568627]], [[0.552941, 0.72549, 0.894118], [0.0352941, 0.254902, 0.54902]]], [[[0.301961, 0.552941, 0.447059], [0.294118, 0.541176, 0.419608]], [[0.898039, 0.266667, 0.137255], [0.854902, 0.603922, 0.0117647]]]], [[0, 1, 0], [0, 1, 0]], 17),\n",
" ([[[[0.0784314, 0.823529, 0.533333], [0.623529, 0.0588235, 0.619608]], [[0.329412, 0.12549, 0.0196078], [0.52549, 0.235294, 0.752941]]], [[[0.462745, 0.180392, 0.211765], [0.52549, 0.0313726, 0.933333]], [[0.305882, 0.760784, 0.360784], [0.12549, 0.639216, 0.52549]]]], [[0, 0, 1], [1, 0, 0]], 18),\n",
" ([[[[0.701961, 0.564706, 0.0588235], [0.737255, 0.760784, 0.921569]], [[0.537255, 0.415686, 0.447059], [0.2, 0.984314, 1.0]]], [[[0.517647, 0.933333, 0.141176], [0.352941, 0.0352941, 0.447059]], [[0.905882, 0.486275, 0.737255], [0.443137, 0.905882, 0.631373]]]], [[1, 0, 0], [1, 0, 0]], 19),\n",
" ([[[[0.878431, 0.0352941, 0.176471], [0.419608, 0.207843, 0.258824]], [[0.243137, 0.741176, 0.882353], [0.298039, 0.356863, 0.207843]]], [[[0.0392157, 0.482353, 0.309804], [0.0509804, 0.737255, 0.768628]], [[0.231373, 0.94902, 0.290196], [0.262745, 0.878431, 0.596078]]]], [[0, 0, 1], [0, 1, 0]], 20),\n",
" ([[[[0.376471, 0.733333, 0.662745], [0.121569, 0.25098, 0.360784]], [[0.686275, 0.309804, 0.0941177], [0.443137, 0.231373, 0.631373]]], [[[0.239216, 0.721569, 0.658824], [0.764706, 0.529412, 0.172549]], [[0.694118, 0.670588, 0.52549], [0.729412, 0.113725, 0.427451]]]], [[1, 0, 0], [0, 1, 0]], 21),\n",
" ([[[[0.85098, 0.866667, 0.113725], [0.478431, 0.572549, 0.152941]], [[0.964706, 0.878431, 0.203922], [0.694118, 0.647059, 0.431373]]], [[[0.67451, 0.831373, 0.839216], [0.67451, 0.752941, 0.713726]], [[0.705882, 0.933333, 0.129412], [0.917647, 0.184314, 0.372549]]]], [[0, 0, 1], [0, 0, 1]], 22),\n",
" ([[[[0.0627451, 0.670588, 0.00392157], [0.270588, 0.0941177, 0.380392]], [[0.0, 0.752941, 0.505882], [0.25098, 0.109804, 0.188235]]], [[[0.937255, 0.694118, 0.513726], [0.529412, 0.713726, 0.752941]], [[0.247059, 0.368627, 0.513726], [0.776471, 0.541176, 0.2]]]], [[0, 0, 1], [0, 0, 1]], 23),\n",
" ([[[[0.203922, 0.627451, 0.792157], [0.835294, 0.482353, 0.462745]], [[0.662745, 0.321569, 0.133333], [0.411765, 0.027451, 0.839216]]], [[[0.0627451, 0.929412, 0.552941], [0.490196, 0.137255, 0.4]], [[0.352941, 0.25098, 0.882353], [0.92549, 0.403922, 0.839216]]]], [[0, 0, 1], [1, 0, 0]], 24),\n",
" ([[[[0.603922, 0.529412, 0.443137], [0.0352941, 0.164706, 0.376471]], [[0.729412, 0.678431, 0.905882], [0.439216, 0.427451, 0.14902]]], [[[0.160784, 0.752941, 0.52549], [0.533333, 0.403922, 0.588235]], [[0.2, 0.407843, 0.858824], [0.290196, 0.788235, 0.858824]]]], [[1, 0, 0], [0, 1, 0]], 25)]"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;\n",
"SELECT madlib.validation_preprocessor_dl(\n",
" 'image_data', -- Source table\n",
" 'val_image_data_packed', -- Output table\n",
" 'species', -- Dependent variable\n",
" 'rgb', -- Independent variable\n",
" 'image_data_packed', -- From training preprocessor step\n",
" 2 -- Buffer size\n",
" );\n",
"SELECT * FROM val_image_data_packed ORDER BY buffer_id;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Review the output summary table:"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>source_table</th>\n",
" <th>output_table</th>\n",
" <th>dependent_varname</th>\n",
" <th>independent_varname</th>\n",
" <th>dependent_vartype</th>\n",
" <th>class_values</th>\n",
" <th>buffer_size</th>\n",
" <th>normalizing_const</th>\n",
" <th>num_classes</th>\n",
" </tr>\n",
" <tr>\n",
" <td>image_data</td>\n",
" <td>val_image_data_packed</td>\n",
" <td>species</td>\n",
" <td>rgb</td>\n",
" <td>text</td>\n",
" <td>[u'bird', u'cat', u'dog']</td>\n",
" <td>2</td>\n",
" <td>255.0</td>\n",
" <td>3</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[(u'image_data', u'val_image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 2, Decimal('255.0'), 3)]"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"SELECT * FROM val_image_data_packed_summary;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"load_data2\"></a>\n",
"# 4. Load data, another format\n",
"Create an artificial 2x2 resolution color image data set with 3 possible classifications. The RGB values are unrolled in to a flat array:"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"52 rows affected.\n",
"52 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>rgb</th>\n",
" <th>species</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[87, 118, 166, 176, 152, 5, 135, 219, 1, 249, 60, 67]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[109, 9, 113, 57, 10, 234, 47, 6, 223, 16, 9, 148]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[82, 94, 197, 145, 99, 28, 7, 8, 203, 159, 13, 83]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[14, 75, 182, 81, 218, 36, 90, 74, 93, 100, 52, 140]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[19, 156, 115, 167, 206, 198, 5, 147, 86, 104, 175, 93]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[122, 252, 125, 205, 118, 140, 24, 44, 221, 242, 80, 55]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[173, 155, 112, 57, 6, 131, 212, 121, 42, 162, 63, 47]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[149, 150, 227, 242, 6, 93, 238, 132, 42, 100, 15, 66]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[236, 52, 225, 36, 112, 141, 191, 224, 198, 197, 98, 154]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[140, 60, 125, 187, 113, 18, 81, 84, 5, 88, 178, 243]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[220, 87, 235, 30, 232, 216, 82, 200, 251, 194, 85, 186]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[27, 127, 5, 181, 189, 145, 241, 58, 76, 97, 76, 157]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[80, 245, 103, 67, 209, 67, 154, 188, 97, 130, 148, 179]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[143, 117, 160, 74, 23, 187, 200, 28, 111, 133, 173, 96]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[249, 193, 10, 150, 119, 91, 139, 222, 158, 92, 33, 56]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[130, 186, 172, 53, 4, 59, 170, 164, 133, 193, 94, 77]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[206, 210, 138, 46, 145, 131, 239, 156, 24, 102, 246, 163]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[148, 255, 101, 204, 23, 231, 134, 195, 27, 138, 254, 197]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[131, 134, 140, 208, 100, 90, 162, 238, 136, 52, 112, 119]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[137, 221, 198, 44, 34, 90, 42, 135, 38, 65, 109, 171]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[136, 54, 1, 78, 99, 132, 212, 239, 84, 56, 73, 246]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[209, 42, 150, 72, 249, 30, 37, 191, 74, 71, 24, 116]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[62, 181, 59, 233, 185, 195, 31, 187, 17, 130, 63, 229]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[147, 28, 187, 137, 66, 140, 179, 215, 211, 172, 246, 249]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[64, 63, 131, 180, 13, 193, 105, 72, 170, 35, 11, 201]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[28, 75, 28, 0, 189, 175, 29, 120, 56, 94, 3, 235]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[215, 151, 44, 207, 2, 107, 15, 133, 31, 28, 71, 137]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[197, 68, 195, 107, 92, 71, 80, 55, 239, 70, 26, 198]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[6, 230, 54, 75, 186, 42, 36, 112, 227, 19, 109, 220]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[145, 117, 8, 147, 175, 205, 215, 113, 57, 51, 184, 136]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[167, 207, 132, 109, 6, 138, 83, 60, 213, 13, 102, 249]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[73, 12, 234, 37, 13, 123, 154, 21, 14, 72, 226, 229]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[26, 24, 114, 163, 130, 25, 114, 6, 134, 119, 144, 217]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[101, 230, 25, 94, 99, 99, 106, 77, 136, 119, 199, 34]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[213, 106, 109, 186, 36, 136, 210, 151, 43, 84, 176, 156]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[54, 20, 235, 15, 199, 80, 245, 224, 174, 87, 67, 24]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[203, 143, 107, 237, 26, 65, 87, 136, 251, 123, 16, 205]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[58, 34, 193, 214, 124, 248, 234, 103, 7, 177, 183, 251]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[101, 83, 212, 125, 247, 159, 12, 98, 139, 38, 163, 226]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[157, 93, 189, 107, 110, 248, 140, 48, 206, 8, 39, 184]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[46, 105, 38, 41, 250, 139, 124, 206, 8, 115, 109, 19]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[248, 57, 119, 218, 231, 21, 55, 164, 127, 166, 156, 11]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[106, 20, 253, 34, 131, 43, 139, 170, 84, 133, 53, 208]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[60, 67, 191, 79, 24, 184, 136, 143, 146, 111, 164, 201]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[35, 110, 175, 47, 68, 25, 67, 65, 59, 198, 107, 198]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[191, 74, 164, 144, 157, 224, 211, 92, 48, 234, 20, 184]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[166, 39, 29, 110, 59, 65, 221, 234, 111, 33, 3, 178]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[62, 120, 205, 4, 231, 140, 78, 139, 28, 235, 108, 238]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[155, 217, 91, 83, 82, 0, 122, 111, 110, 181, 176, 75]</td>\n",
" <td>dog</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[31, 108, 162, 209, 205, 224, 73, 154, 228, 48, 38, 50]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[65, 29, 39, 48, 100, 194, 8, 190, 21, 90, 190, 144]</td>\n",
" <td>bird</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[45, 69, 122, 120, 228, 153, 228, 134, 106, 177, 103, 179]</td>\n",
" <td>cat</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([87, 118, 166, 176, 152, 5, 135, 219, 1, 249, 60, 67], u'dog'),\n",
" ([109, 9, 113, 57, 10, 234, 47, 6, 223, 16, 9, 148], u'cat'),\n",
" ([82, 94, 197, 145, 99, 28, 7, 8, 203, 159, 13, 83], u'dog'),\n",
" ([14, 75, 182, 81, 218, 36, 90, 74, 93, 100, 52, 140], u'dog'),\n",
" ([19, 156, 115, 167, 206, 198, 5, 147, 86, 104, 175, 93], u'dog'),\n",
" ([122, 252, 125, 205, 118, 140, 24, 44, 221, 242, 80, 55], u'cat'),\n",
" ([173, 155, 112, 57, 6, 131, 212, 121, 42, 162, 63, 47], u'cat'),\n",
" ([149, 150, 227, 242, 6, 93, 238, 132, 42, 100, 15, 66], u'dog'),\n",
" ([236, 52, 225, 36, 112, 141, 191, 224, 198, 197, 98, 154], u'cat'),\n",
" ([140, 60, 125, 187, 113, 18, 81, 84, 5, 88, 178, 243], u'bird'),\n",
" ([220, 87, 235, 30, 232, 216, 82, 200, 251, 194, 85, 186], u'dog'),\n",
" ([27, 127, 5, 181, 189, 145, 241, 58, 76, 97, 76, 157], u'bird'),\n",
" ([80, 245, 103, 67, 209, 67, 154, 188, 97, 130, 148, 179], u'cat'),\n",
" ([143, 117, 160, 74, 23, 187, 200, 28, 111, 133, 173, 96], u'bird'),\n",
" ([249, 193, 10, 150, 119, 91, 139, 222, 158, 92, 33, 56], u'cat'),\n",
" ([130, 186, 172, 53, 4, 59, 170, 164, 133, 193, 94, 77], u'bird'),\n",
" ([206, 210, 138, 46, 145, 131, 239, 156, 24, 102, 246, 163], u'cat'),\n",
" ([148, 255, 101, 204, 23, 231, 134, 195, 27, 138, 254, 197], u'cat'),\n",
" ([131, 134, 140, 208, 100, 90, 162, 238, 136, 52, 112, 119], u'bird'),\n",
" ([137, 221, 198, 44, 34, 90, 42, 135, 38, 65, 109, 171], u'cat'),\n",
" ([136, 54, 1, 78, 99, 132, 212, 239, 84, 56, 73, 246], u'cat'),\n",
" ([209, 42, 150, 72, 249, 30, 37, 191, 74, 71, 24, 116], u'bird'),\n",
" ([62, 181, 59, 233, 185, 195, 31, 187, 17, 130, 63, 229], u'dog'),\n",
" ([147, 28, 187, 137, 66, 140, 179, 215, 211, 172, 246, 249], u'dog'),\n",
" ([64, 63, 131, 180, 13, 193, 105, 72, 170, 35, 11, 201], u'bird'),\n",
" ([28, 75, 28, 0, 189, 175, 29, 120, 56, 94, 3, 235], u'cat'),\n",
" ([215, 151, 44, 207, 2, 107, 15, 133, 31, 28, 71, 137], u'dog'),\n",
" ([197, 68, 195, 107, 92, 71, 80, 55, 239, 70, 26, 198], u'cat'),\n",
" ([6, 230, 54, 75, 186, 42, 36, 112, 227, 19, 109, 220], u'cat'),\n",
" ([145, 117, 8, 147, 175, 205, 215, 113, 57, 51, 184, 136], u'dog'),\n",
" ([167, 207, 132, 109, 6, 138, 83, 60, 213, 13, 102, 249], u'dog'),\n",
" ([73, 12, 234, 37, 13, 123, 154, 21, 14, 72, 226, 229], u'bird'),\n",
" ([26, 24, 114, 163, 130, 25, 114, 6, 134, 119, 144, 217], u'bird'),\n",
" ([101, 230, 25, 94, 99, 99, 106, 77, 136, 119, 199, 34], u'dog'),\n",
" ([213, 106, 109, 186, 36, 136, 210, 151, 43, 84, 176, 156], u'dog'),\n",
" ([54, 20, 235, 15, 199, 80, 245, 224, 174, 87, 67, 24], u'dog'),\n",
" ([203, 143, 107, 237, 26, 65, 87, 136, 251, 123, 16, 205], u'cat'),\n",
" ([58, 34, 193, 214, 124, 248, 234, 103, 7, 177, 183, 251], u'dog'),\n",
" ([101, 83, 212, 125, 247, 159, 12, 98, 139, 38, 163, 226], u'bird'),\n",
" ([157, 93, 189, 107, 110, 248, 140, 48, 206, 8, 39, 184], u'dog'),\n",
" ([46, 105, 38, 41, 250, 139, 124, 206, 8, 115, 109, 19], u'bird'),\n",
" ([248, 57, 119, 218, 231, 21, 55, 164, 127, 166, 156, 11], u'bird'),\n",
" ([106, 20, 253, 34, 131, 43, 139, 170, 84, 133, 53, 208], u'cat'),\n",
" ([60, 67, 191, 79, 24, 184, 136, 143, 146, 111, 164, 201], u'cat'),\n",
" ([35, 110, 175, 47, 68, 25, 67, 65, 59, 198, 107, 198], u'dog'),\n",
" ([191, 74, 164, 144, 157, 224, 211, 92, 48, 234, 20, 184], u'dog'),\n",
" ([166, 39, 29, 110, 59, 65, 221, 234, 111, 33, 3, 178], u'dog'),\n",
" ([62, 120, 205, 4, 231, 140, 78, 139, 28, 235, 108, 238], u'cat'),\n",
" ([155, 217, 91, 83, 82, 0, 122, 111, 110, 181, 176, 75], u'dog'),\n",
" ([31, 108, 162, 209, 205, 224, 73, 154, 228, 48, 38, 50], u'bird'),\n",
" ([65, 29, 39, 48, 100, 194, 8, 190, 21, 90, 190, 144], u'bird'),\n",
" ([45, 69, 122, 120, 228, 153, 228, 134, 106, 177, 103, 179], u'cat')]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS image_data;\n",
"\n",
"CREATE TABLE image_data AS (\n",
"SELECT ARRAY[\n",
" (random() * 256)::integer, -- R values\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer, -- G values\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer, -- B values\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer,\n",
" (random() * 256)::integer\n",
" ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species\n",
"FROM generate_series(1, 52)\n",
");\n",
"\n",
"SELECT * FROM image_data;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"pp_train2\"></a>\n",
"# 5. Run preprocessor for training image data\n",
"\n",
"Run the preprocessor to generate the packed output table:"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"1 rows affected.\n",
"2 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>independent_var</th>\n",
" <th>dependent_var</th>\n",
" <th>buffer_id</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.105882, 0.498039, 0.0196078, 0.709804, 0.741176, 0.568627, 0.945098, 0.227451, 0.298039, 0.380392, 0.298039, 0.615686], [0.537255, 0.866667, 0.776471, 0.172549, 0.133333, 0.352941, 0.164706, 0.529412, 0.14902, 0.254902, 0.427451, 0.670588], [0.235294, 0.262745, 0.74902, 0.309804, 0.0941177, 0.721569, 0.533333, 0.560784, 0.572549, 0.435294, 0.643137, 0.788235], [0.560784, 0.458824, 0.627451, 0.290196, 0.0901961, 0.733333, 0.784314, 0.109804, 0.435294, 0.521569, 0.678431, 0.376471], [0.576471, 0.109804, 0.733333, 0.537255, 0.258824, 0.54902, 0.701961, 0.843137, 0.827451, 0.67451, 0.964706, 0.976471], [0.427451, 0.0352941, 0.443137, 0.223529, 0.0392157, 0.917647, 0.184314, 0.0235294, 0.87451, 0.0627451, 0.0352941, 0.580392], [0.396078, 0.901961, 0.0980392, 0.368627, 0.388235, 0.388235, 0.415686, 0.301961, 0.533333, 0.466667, 0.780392, 0.133333], [0.121569, 0.423529, 0.635294, 0.819608, 0.803922, 0.878431, 0.286275, 0.603922, 0.894118, 0.188235, 0.14902, 0.196078], [0.568627, 0.458824, 0.0313726, 0.576471, 0.686275, 0.803922, 0.843137, 0.443137, 0.223529, 0.2, 0.721569, 0.533333], [0.772549, 0.266667, 0.764706, 0.419608, 0.360784, 0.278431, 0.313726, 0.215686, 0.937255, 0.27451, 0.101961, 0.776471], [0.54902, 0.235294, 0.490196, 0.733333, 0.443137, 0.0705882, 0.317647, 0.329412, 0.0196078, 0.345098, 0.698039, 0.952941], [0.580392, 1.0, 0.396078, 0.8, 0.0901961, 0.905882, 0.52549, 0.764706, 0.105882, 0.541176, 0.996078, 0.772549], [0.584314, 0.588235, 0.890196, 0.94902, 0.0235294, 0.364706, 0.933333, 0.517647, 0.164706, 0.392157, 0.0588235, 0.258824], [0.819608, 0.164706, 0.588235, 0.282353, 0.976471, 0.117647, 0.145098, 0.74902, 0.290196, 0.278431, 0.0941177, 0.454902], [0.615686, 0.364706, 0.741176, 0.419608, 0.431373, 0.972549, 0.54902, 0.188235, 0.807843, 0.0313726, 0.152941, 0.721569], [0.176471, 0.270588, 0.478431, 0.470588, 0.894118, 0.6, 0.894118, 0.52549, 0.415686, 0.694118, 0.403922, 0.701961], [0.109804, 0.294118, 0.109804, 0.0, 0.741176, 0.686275, 0.113725, 0.470588, 0.219608, 0.368627, 0.0117647, 0.921569], [0.243137, 0.470588, 0.803922, 0.0156863, 0.905882, 0.54902, 0.305882, 0.545098, 0.109804, 0.921569, 0.423529, 0.933333], [0.286275, 0.0470588, 0.917647, 0.145098, 0.0509804, 0.482353, 0.603922, 0.0823529, 0.054902, 0.282353, 0.886275, 0.898039], [0.972549, 0.223529, 0.466667, 0.854902, 0.905882, 0.0823529, 0.215686, 0.643137, 0.498039, 0.65098, 0.611765, 0.0431373], [0.509804, 0.729412, 0.67451, 0.207843, 0.0156863, 0.231373, 0.666667, 0.643137, 0.521569, 0.756863, 0.368627, 0.301961], [0.227451, 0.133333, 0.756863, 0.839216, 0.486275, 0.972549, 0.917647, 0.403922, 0.027451, 0.694118, 0.717647, 0.984314], [0.74902, 0.290196, 0.643137, 0.564706, 0.615686, 0.878431, 0.827451, 0.360784, 0.188235, 0.917647, 0.0784314, 0.721569], [0.211765, 0.0784314, 0.921569, 0.0588235, 0.780392, 0.313726, 0.960784, 0.878431, 0.682353, 0.341176, 0.262745, 0.0941177], [0.054902, 0.294118, 0.713726, 0.317647, 0.854902, 0.141176, 0.352941, 0.290196, 0.364706, 0.392157, 0.203922, 0.54902], [0.478431, 0.988235, 0.490196, 0.803922, 0.462745, 0.54902, 0.0941177, 0.172549, 0.866667, 0.94902, 0.313726, 0.215686]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0]]</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.313726, 0.960784, 0.403922, 0.262745, 0.819608, 0.262745, 0.603922, 0.737255, 0.380392, 0.509804, 0.580392, 0.701961], [0.654902, 0.811765, 0.517647, 0.427451, 0.0235294, 0.541176, 0.32549, 0.235294, 0.835294, 0.0509804, 0.4, 0.976471], [0.180392, 0.411765, 0.14902, 0.160784, 0.980392, 0.545098, 0.486275, 0.807843, 0.0313726, 0.45098, 0.427451, 0.0745098], [0.0235294, 0.901961, 0.211765, 0.294118, 0.729412, 0.164706, 0.141176, 0.439216, 0.890196, 0.0745098, 0.427451, 0.862745], [0.321569, 0.368627, 0.772549, 0.568627, 0.388235, 0.109804, 0.027451, 0.0313726, 0.796079, 0.623529, 0.0509804, 0.32549], [0.796079, 0.560784, 0.419608, 0.929412, 0.101961, 0.254902, 0.341176, 0.533333, 0.984314, 0.482353, 0.0627451, 0.803922], [0.607843, 0.85098, 0.356863, 0.32549, 0.321569, 0.0, 0.478431, 0.435294, 0.431373, 0.709804, 0.690196, 0.294118], [0.678431, 0.607843, 0.439216, 0.223529, 0.0235294, 0.513726, 0.831373, 0.47451, 0.164706, 0.635294, 0.247059, 0.184314], [0.533333, 0.211765, 0.00392157, 0.305882, 0.388235, 0.517647, 0.831373, 0.937255, 0.329412, 0.219608, 0.286275, 0.964706], [0.92549, 0.203922, 0.882353, 0.141176, 0.439216, 0.552941, 0.74902, 0.878431, 0.776471, 0.772549, 0.384314, 0.603922], [0.513726, 0.52549, 0.54902, 0.815686, 0.392157, 0.352941, 0.635294, 0.933333, 0.533333, 0.203922, 0.439216, 0.466667], [0.976471, 0.756863, 0.0392157, 0.588235, 0.466667, 0.356863, 0.545098, 0.870588, 0.619608, 0.360784, 0.129412, 0.219608], [0.396078, 0.32549, 0.831373, 0.490196, 0.968628, 0.623529, 0.0470588, 0.384314, 0.545098, 0.14902, 0.639216, 0.886275], [0.862745, 0.341176, 0.921569, 0.117647, 0.909804, 0.847059, 0.321569, 0.784314, 0.984314, 0.760784, 0.333333, 0.729412], [0.101961, 0.0941177, 0.447059, 0.639216, 0.509804, 0.0980392, 0.447059, 0.0235294, 0.52549, 0.466667, 0.564706, 0.85098], [0.843137, 0.592157, 0.172549, 0.811765, 0.00784314, 0.419608, 0.0588235, 0.521569, 0.121569, 0.109804, 0.278431, 0.537255], [0.341176, 0.462745, 0.65098, 0.690196, 0.596078, 0.0196078, 0.529412, 0.858824, 0.00392157, 0.976471, 0.235294, 0.262745], [0.0745098, 0.611765, 0.45098, 0.654902, 0.807843, 0.776471, 0.0196078, 0.576471, 0.337255, 0.407843, 0.686275, 0.364706], [0.415686, 0.0784314, 0.992157, 0.133333, 0.513726, 0.168627, 0.545098, 0.666667, 0.329412, 0.521569, 0.207843, 0.815686], [0.835294, 0.415686, 0.427451, 0.729412, 0.141176, 0.533333, 0.823529, 0.592157, 0.168627, 0.329412, 0.690196, 0.611765], [0.254902, 0.113725, 0.152941, 0.188235, 0.392157, 0.760784, 0.0313726, 0.745098, 0.0823529, 0.352941, 0.745098, 0.564706], [0.807843, 0.823529, 0.541176, 0.180392, 0.568627, 0.513726, 0.937255, 0.611765, 0.0941177, 0.4, 0.964706, 0.639216], [0.243137, 0.709804, 0.231373, 0.913726, 0.72549, 0.764706, 0.121569, 0.733333, 0.0666667, 0.509804, 0.247059, 0.898039], [0.65098, 0.152941, 0.113725, 0.431373, 0.231373, 0.254902, 0.866667, 0.917647, 0.435294, 0.129412, 0.0117647, 0.698039], [0.137255, 0.431373, 0.686275, 0.184314, 0.266667, 0.0980392, 0.262745, 0.254902, 0.231373, 0.776471, 0.419608, 0.776471], [0.25098, 0.247059, 0.513726, 0.705882, 0.0509804, 0.756863, 0.411765, 0.282353, 0.666667, 0.137255, 0.0431373, 0.788235]]</td>\n",
" <td>[[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0]]</td>\n",
" <td>1</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[0.105882, 0.498039, 0.0196078, 0.709804, 0.741176, 0.568627, 0.945098, 0.227451, 0.298039, 0.380392, 0.298039, 0.615686], [0.537255, 0.866667, 0.776471, 0.172549, 0.133333, 0.352941, 0.164706, 0.529412, 0.14902, 0.254902, 0.427451, 0.670588], [0.235294, 0.262745, 0.74902, 0.309804, 0.0941177, 0.721569, 0.533333, 0.560784, 0.572549, 0.435294, 0.643137, 0.788235], [0.560784, 0.458824, 0.627451, 0.290196, 0.0901961, 0.733333, 0.784314, 0.109804, 0.435294, 0.521569, 0.678431, 0.376471], [0.576471, 0.109804, 0.733333, 0.537255, 0.258824, 0.54902, 0.701961, 0.843137, 0.827451, 0.67451, 0.964706, 0.976471], [0.427451, 0.0352941, 0.443137, 0.223529, 0.0392157, 0.917647, 0.184314, 0.0235294, 0.87451, 0.0627451, 0.0352941, 0.580392], [0.396078, 0.901961, 0.0980392, 0.368627, 0.388235, 0.388235, 0.415686, 0.301961, 0.533333, 0.466667, 0.780392, 0.133333], [0.121569, 0.423529, 0.635294, 0.819608, 0.803922, 0.878431, 0.286275, 0.603922, 0.894118, 0.188235, 0.14902, 0.196078], [0.568627, 0.458824, 0.0313726, 0.576471, 0.686275, 0.803922, 0.843137, 0.443137, 0.223529, 0.2, 0.721569, 0.533333], [0.772549, 0.266667, 0.764706, 0.419608, 0.360784, 0.278431, 0.313726, 0.215686, 0.937255, 0.27451, 0.101961, 0.776471], [0.54902, 0.235294, 0.490196, 0.733333, 0.443137, 0.0705882, 0.317647, 0.329412, 0.0196078, 0.345098, 0.698039, 0.952941], [0.580392, 1.0, 0.396078, 0.8, 0.0901961, 0.905882, 0.52549, 0.764706, 0.105882, 0.541176, 0.996078, 0.772549], [0.584314, 0.588235, 0.890196, 0.94902, 0.0235294, 0.364706, 0.933333, 0.517647, 0.164706, 0.392157, 0.0588235, 0.258824], [0.819608, 0.164706, 0.588235, 0.282353, 0.976471, 0.117647, 0.145098, 0.74902, 0.290196, 0.278431, 0.0941177, 0.454902], [0.615686, 0.364706, 0.741176, 0.419608, 0.431373, 0.972549, 0.54902, 0.188235, 0.807843, 0.0313726, 0.152941, 0.721569], [0.176471, 0.270588, 0.478431, 0.470588, 0.894118, 0.6, 0.894118, 0.52549, 0.415686, 0.694118, 0.403922, 0.701961], [0.109804, 0.294118, 0.109804, 0.0, 0.741176, 0.686275, 0.113725, 0.470588, 0.219608, 0.368627, 0.0117647, 0.921569], [0.243137, 0.470588, 0.803922, 0.0156863, 0.905882, 0.54902, 0.305882, 0.545098, 0.109804, 0.921569, 0.423529, 0.933333], [0.286275, 0.0470588, 0.917647, 0.145098, 0.0509804, 0.482353, 0.603922, 0.0823529, 0.054902, 0.282353, 0.886275, 0.898039], [0.972549, 0.223529, 0.466667, 0.854902, 0.905882, 0.0823529, 0.215686, 0.643137, 0.498039, 0.65098, 0.611765, 0.0431373], [0.509804, 0.729412, 0.67451, 0.207843, 0.0156863, 0.231373, 0.666667, 0.643137, 0.521569, 0.756863, 0.368627, 0.301961], [0.227451, 0.133333, 0.756863, 0.839216, 0.486275, 0.972549, 0.917647, 0.403922, 0.027451, 0.694118, 0.717647, 0.984314], [0.74902, 0.290196, 0.643137, 0.564706, 0.615686, 0.878431, 0.827451, 0.360784, 0.188235, 0.917647, 0.0784314, 0.721569], [0.211765, 0.0784314, 0.921569, 0.0588235, 0.780392, 0.313726, 0.960784, 0.878431, 0.682353, 0.341176, 0.262745, 0.0941177], [0.054902, 0.294118, 0.713726, 0.317647, 0.854902, 0.141176, 0.352941, 0.290196, 0.364706, 0.392157, 0.203922, 0.54902], [0.478431, 0.988235, 0.490196, 0.803922, 0.462745, 0.54902, 0.0941177, 0.172549, 0.866667, 0.94902, 0.313726, 0.215686]], [[1, 0, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0]], 0),\n",
" ([[0.313726, 0.960784, 0.403922, 0.262745, 0.819608, 0.262745, 0.603922, 0.737255, 0.380392, 0.509804, 0.580392, 0.701961], [0.654902, 0.811765, 0.517647, 0.427451, 0.0235294, 0.541176, 0.32549, 0.235294, 0.835294, 0.0509804, 0.4, 0.976471], [0.180392, 0.411765, 0.14902, 0.160784, 0.980392, 0.545098, 0.486275, 0.807843, 0.0313726, 0.45098, 0.427451, 0.0745098], [0.0235294, 0.901961, 0.211765, 0.294118, 0.729412, 0.164706, 0.141176, 0.439216, 0.890196, 0.0745098, 0.427451, 0.862745], [0.321569, 0.368627, 0.772549, 0.568627, 0.388235, 0.109804, 0.027451, 0.0313726, 0.796079, 0.623529, 0.0509804, 0.32549], [0.796079, 0.560784, 0.419608, 0.929412, 0.101961, 0.254902, 0.341176, 0.533333, 0.984314, 0.482353, 0.0627451, 0.803922], [0.607843, 0.85098, 0.356863, 0.32549, 0.321569, 0.0, 0.478431, 0.435294, 0.431373, 0.709804, 0.690196, 0.294118], [0.678431, 0.607843, 0.439216, 0.223529, 0.0235294, 0.513726, 0.831373, 0.47451, 0.164706, 0.635294, 0.247059, 0.184314], [0.533333, 0.211765, 0.00392157, 0.305882, 0.388235, 0.517647, 0.831373, 0.937255, 0.329412, 0.219608, 0.286275, 0.964706], [0.92549, 0.203922, 0.882353, 0.141176, 0.439216, 0.552941, 0.74902, 0.878431, 0.776471, 0.772549, 0.384314, 0.603922], [0.513726, 0.52549, 0.54902, 0.815686, 0.392157, 0.352941, 0.635294, 0.933333, 0.533333, 0.203922, 0.439216, 0.466667], [0.976471, 0.756863, 0.0392157, 0.588235, 0.466667, 0.356863, 0.545098, 0.870588, 0.619608, 0.360784, 0.129412, 0.219608], [0.396078, 0.32549, 0.831373, 0.490196, 0.968628, 0.623529, 0.0470588, 0.384314, 0.545098, 0.14902, 0.639216, 0.886275], [0.862745, 0.341176, 0.921569, 0.117647, 0.909804, 0.847059, 0.321569, 0.784314, 0.984314, 0.760784, 0.333333, 0.729412], [0.101961, 0.0941177, 0.447059, 0.639216, 0.509804, 0.0980392, 0.447059, 0.0235294, 0.52549, 0.466667, 0.564706, 0.85098], [0.843137, 0.592157, 0.172549, 0.811765, 0.00784314, 0.419608, 0.0588235, 0.521569, 0.121569, 0.109804, 0.278431, 0.537255], [0.341176, 0.462745, 0.65098, 0.690196, 0.596078, 0.0196078, 0.529412, 0.858824, 0.00392157, 0.976471, 0.235294, 0.262745], [0.0745098, 0.611765, 0.45098, 0.654902, 0.807843, 0.776471, 0.0196078, 0.576471, 0.337255, 0.407843, 0.686275, 0.364706], [0.415686, 0.0784314, 0.992157, 0.133333, 0.513726, 0.168627, 0.545098, 0.666667, 0.329412, 0.521569, 0.207843, 0.815686], [0.835294, 0.415686, 0.427451, 0.729412, 0.141176, 0.533333, 0.823529, 0.592157, 0.168627, 0.329412, 0.690196, 0.611765], [0.254902, 0.113725, 0.152941, 0.188235, 0.392157, 0.760784, 0.0313726, 0.745098, 0.0823529, 0.352941, 0.745098, 0.564706], [0.807843, 0.823529, 0.541176, 0.180392, 0.568627, 0.513726, 0.937255, 0.611765, 0.0941177, 0.4, 0.964706, 0.639216], [0.243137, 0.709804, 0.231373, 0.913726, 0.72549, 0.764706, 0.121569, 0.733333, 0.0666667, 0.509804, 0.247059, 0.898039], [0.65098, 0.152941, 0.113725, 0.431373, 0.231373, 0.254902, 0.866667, 0.917647, 0.435294, 0.129412, 0.0117647, 0.698039], [0.137255, 0.431373, 0.686275, 0.184314, 0.266667, 0.0980392, 0.262745, 0.254902, 0.231373, 0.776471, 0.419608, 0.776471], [0.25098, 0.247059, 0.513726, 0.705882, 0.0509804, 0.756863, 0.411765, 0.282353, 0.666667, 0.137255, 0.0431373, 0.788235]], [[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0]], 1)]"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
"\n",
"SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
" 'image_data_packed', -- Output table\n",
" 'species', -- Dependent variable\n",
" 'rgb', -- Independent variable\n",
" NULL, -- Buffer size\n",
" 255 -- Normalizing constant\n",
" );\n",
"\n",
"SELECT * FROM image_data_packed ORDER BY buffer_id;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"pp_val2\"></a>\n",
"# 6. Run preprocessor for validation image data\n",
"\n",
"Run the preprocessor for the validation dataset. In this example, we use the same images for validation to demonstrate, but normally validation data is different than training data:"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"1 rows affected.\n",
"2 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>independent_var</th>\n",
" <th>dependent_var</th>\n",
" <th>buffer_id</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.843137, 0.85098, 0.972549, 0.92549, 0.227451, 0.980392, 0.823529, 0.388235, 0.631373, 0.00784314, 0.701961, 0.14902], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.00392157, 0.231373, 0.941177, 0.6, 0.364706, 0.419608, 0.811765, 0.243137, 0.745098, 0.552941, 0.968628, 0.913726], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.286275], [0.0666667, 0.101961, 0.827451, 0.313726, 0.168627, 0.972549, 0.521569, 0.0431373, 0.227451, 0.376471, 0.929412, 0.717647], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.835294, 0.690196, 0.639216, 0.227451, 0.372549, 0.294118, 0.0509804, 0.203922, 0.756863, 0.815686, 0.956863, 0.564706], [0.278431, 0.854902, 0.623529, 0.184314, 0.270588, 0.45098, 0.870588, 0.909804, 0.682353, 0.239216, 0.2, 0.733333], [0.219608, 0.933333, 0.223529, 0.145098, 0.443137, 0.505882, 1.0, 0.0627451, 0.690196, 0.266667, 0.513726, 0.556863], [0.635294, 0.337255, 0.419608, 0.607843, 0.780392, 0.639216, 0.541176, 0.00392157, 0.784314, 0.984314, 0.509804, 0.776471], [0.764706, 0.866667, 0.486275, 0.913726, 0.517647, 0.113725, 0.247059, 0.937255, 0.72549, 0.0235294, 0.572549, 0.258824], [0.254902, 0.792157, 0.87451, 0.396078, 0.192157, 0.635294, 0.254902, 0.67451, 0.545098, 0.772549, 0.788235, 0.792157], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0.454902], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.407843, 0.607843, 0.152941, 0.913726, 0.972549, 0.298039, 0.588235, 0.486275, 0.321569, 0.054902, 0.52549, 0.0745098]]</td>\n",
" <td>[[0, 1, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1]]</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098], [0.333333, 0.764706, 0.862745, 0.611765, 0.223529, 0.737255, 0.647059, 0.917647, 0.0901961, 0.00784314, 0.439216, 0.0823529], [0.603922, 0.207843, 0.254902, 0.635294, 0.160784, 0.592157, 0.396078, 0.0156863, 0.196078, 0.619608, 0.752941, 0.843137], [0.933333, 0.0470588, 0.964706, 0.752941, 0.443137, 0.564706, 0.960784, 0.701961, 0.196078, 0.113725, 0.286275, 0.596078], [0.317647, 0.470588, 0.486275, 0.466667, 0.282353, 0.411765, 0.513726, 0.247059, 0.160784, 0.956863, 0.811765, 0.113725], [0.607843, 0.309804, 0.956863, 0.0705882, 0.0901961, 0.27451, 0.545098, 0.576471, 0.741176, 0.827451, 0.988235, 0.25098], [0.960784, 0.839216, 0.0588235, 0.870588, 0.101961, 0.666667, 0.176471, 0.054902, 0.737255, 0.266667, 0.329412, 0.278431], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [0.501961, 0.278431, 0.635294, 0.215686, 0.45098, 0.0313726, 0.780392, 0.835294, 0.721569, 0.435294, 0.172549, 0.239216], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.34902, 0.239216, 0.537255, 0.12549, 0.282353, 0.729412, 0.164706, 0.839216, 0.478431, 0.376471, 0.588235, 0.0156863], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.286275, 0.388235, 0.0352941, 0.0745098, 0.0862745, 0.545098, 0.890196, 0.360784, 0.309804, 0.733333, 0.984314, 0.317647], [0.768628, 0.345098, 0.00392157, 0.380392, 0.592157, 0.290196, 0.768628, 0.627451, 0.368627, 0.854902, 0.168627, 0.254902], [0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.584314, 0.760784, 0.227451, 0.0313726, 0.32549, 0.694118, 0.639216, 0.294118, 0.929412, 0.498039, 0.027451, 0.505882], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
" <td>1</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.843137, 0.85098, 0.972549, 0.92549, 0.227451, 0.980392, 0.823529, 0.388235, 0.631373, 0.00784314, 0.701961, 0.14902], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.00392157, 0.231373, 0.941177, 0.6, 0.364706, 0.419608, 0.811765, 0.243137, 0.745098, 0.552941, 0.968628, 0.913726], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.286275], [0.0666667, 0.101961, 0.827451, 0.313726, 0.168627, 0.972549, 0.521569, 0.0431373, 0.227451, 0.376471, 0.929412, 0.717647], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.835294, 0.690196, 0.639216, 0.227451, 0.372549, 0.294118, 0.0509804, 0.203922, 0.756863, 0.815686, 0.956863, 0.564706], [0.278431, 0.854902, 0.623529, 0.184314, 0.270588, 0.45098, 0.870588, 0.909804, 0.682353, 0.239216, 0.2, 0.733333], [0.219608, 0.933333, 0.223529, 0.145098, 0.443137, 0.505882, 1.0, 0.0627451, 0.690196, 0.266667, 0.513726, 0.556863], [0.635294, 0.337255, 0.419608, 0.607843, 0.780392, 0.639216, 0.541176, 0.00392157, 0.784314, 0.984314, 0.509804, 0.776471], [0.764706, 0.866667, 0.486275, 0.913726, 0.517647, 0.113725, 0.247059, 0.937255, 0.72549, 0.0235294, 0.572549, 0.258824], [0.254902, 0.792157, 0.87451, 0.396078, 0.192157, 0.635294, 0.254902, 0.67451, 0.545098, 0.772549, 0.788235, 0.792157], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0.454902], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.407843, 0.607843, 0.152941, 0.913726, 0.972549, 0.298039, 0.588235, 0.486275, 0.321569, 0.054902, 0.52549, 0.0745098]], [[0, 1, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1]], 0),\n",
" ([[0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098], [0.333333, 0.764706, 0.862745, 0.611765, 0.223529, 0.737255, 0.647059, 0.917647, 0.0901961, 0.00784314, 0.439216, 0.0823529], [0.603922, 0.207843, 0.254902, 0.635294, 0.160784, 0.592157, 0.396078, 0.0156863, 0.196078, 0.619608, 0.752941, 0.843137], [0.933333, 0.0470588, 0.964706, 0.752941, 0.443137, 0.564706, 0.960784, 0.701961, 0.196078, 0.113725, 0.286275, 0.596078], [0.317647, 0.470588, 0.486275, 0.466667, 0.282353, 0.411765, 0.513726, 0.247059, 0.160784, 0.956863, 0.811765, 0.113725], [0.607843, 0.309804, 0.956863, 0.0705882, 0.0901961, 0.27451, 0.545098, 0.576471, 0.741176, 0.827451, 0.988235, 0.25098], [0.960784, 0.839216, 0.0588235, 0.870588, 0.101961, 0.666667, 0.176471, 0.054902, 0.737255, 0.266667, 0.329412, 0.278431], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [0.501961, 0.278431, 0.635294, 0.215686, 0.45098, 0.0313726, 0.780392, 0.835294, 0.721569, 0.435294, 0.172549, 0.239216], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.34902, 0.239216, 0.537255, 0.12549, 0.282353, 0.729412, 0.164706, 0.839216, 0.478431, 0.376471, 0.588235, 0.0156863], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.286275, 0.388235, 0.0352941, 0.0745098, 0.0862745, 0.545098, 0.890196, 0.360784, 0.309804, 0.733333, 0.984314, 0.317647], [0.768628, 0.345098, 0.00392157, 0.380392, 0.592157, 0.290196, 0.768628, 0.627451, 0.368627, 0.854902, 0.168627, 0.254902], [0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.584314, 0.760784, 0.227451, 0.0313726, 0.32549, 0.694118, 0.639216, 0.294118, 0.929412, 0.498039, 0.027451, 0.505882], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804]], [[1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1], [0, 1, 0]], 1)]"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;\n",
"\n",
"SELECT madlib.validation_preprocessor_dl(\n",
" 'image_data', -- Source table\n",
" 'val_image_data_packed', -- Output table\n",
" 'species', -- Dependent variable\n",
" 'rgb', -- Independent variable\n",
" 'image_data_packed', -- From training preprocessor step\n",
" NULL -- Buffer size\n",
" );\n",
"\n",
"SELECT * FROM val_image_data_packed ORDER BY buffer_id;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"change_buffer\"></a>\n",
"# 7. Change buffer size \n",
"\n",
"Generally the default buffer size will work well, but if you have occasion to change it:"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"1 rows affected.\n",
"6 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>independent_var</th>\n",
" <th>dependent_var</th>\n",
" <th>buffer_id</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.286275], [0.835294, 0.690196, 0.639216, 0.227451, 0.372549, 0.294118, 0.0509804, 0.203922, 0.756863, 0.815686, 0.956863, 0.564706], [0.635294, 0.337255, 0.419608, 0.607843, 0.780392, 0.639216, 0.541176, 0.00392157, 0.784314, 0.984314, 0.509804, 0.776471], [0.843137, 0.85098, 0.972549, 0.92549, 0.227451, 0.980392, 0.823529, 0.388235, 0.631373, 0.00784314, 0.701961, 0.14902], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686]]</td>\n",
" <td>[[0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0]]</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.286275, 0.388235, 0.0352941, 0.0745098, 0.0862745, 0.545098, 0.890196, 0.360784, 0.309804, 0.733333, 0.984314, 0.317647], [0.960784, 0.839216, 0.0588235, 0.870588, 0.101961, 0.666667, 0.176471, 0.054902, 0.737255, 0.266667, 0.329412, 0.278431], [0.933333, 0.0470588, 0.964706, 0.752941, 0.443137, 0.564706, 0.960784, 0.701961, 0.196078, 0.113725, 0.286275, 0.596078], [0.584314, 0.760784, 0.227451, 0.0313726, 0.32549, 0.694118, 0.639216, 0.294118, 0.929412, 0.498039, 0.027451, 0.505882], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098]]</td>\n",
" <td>[[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.0666667, 0.101961, 0.827451, 0.313726, 0.168627, 0.972549, 0.521569, 0.0431373, 0.227451, 0.376471, 0.929412, 0.717647], [0.764706, 0.866667, 0.486275, 0.913726, 0.517647, 0.113725, 0.247059, 0.937255, 0.72549, 0.0235294, 0.572549, 0.258824], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.278431, 0.854902, 0.623529, 0.184314, 0.270588, 0.45098, 0.870588, 0.909804, 0.682353, 0.239216, 0.2, 0.733333]]</td>\n",
" <td>[[0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1]]</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [0.317647, 0.470588, 0.486275, 0.466667, 0.282353, 0.411765, 0.513726, 0.247059, 0.160784, 0.956863, 0.811765, 0.113725], [0.34902, 0.239216, 0.537255, 0.12549, 0.282353, 0.729412, 0.164706, 0.839216, 0.478431, 0.376471, 0.588235, 0.0156863], [0.768628, 0.345098, 0.00392157, 0.380392, 0.592157, 0.290196, 0.768628, 0.627451, 0.368627, 0.854902, 0.168627, 0.254902], [0.333333, 0.764706, 0.862745, 0.611765, 0.223529, 0.737255, 0.647059, 0.917647, 0.0901961, 0.00784314, 0.439216, 0.0823529], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314]]</td>\n",
" <td>[[1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]]</td>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0.454902], [0.00392157, 0.231373, 0.941177, 0.6, 0.364706, 0.419608, 0.811765, 0.243137, 0.745098, 0.552941, 0.968628, 0.913726], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.219608, 0.933333, 0.223529, 0.145098, 0.443137, 0.505882, 1.0, 0.0627451, 0.690196, 0.266667, 0.513726, 0.556863], [0.254902, 0.792157, 0.87451, 0.396078, 0.192157, 0.635294, 0.254902, 0.67451, 0.545098, 0.772549, 0.788235, 0.792157], [0.407843, 0.607843, 0.152941, 0.913726, 0.972549, 0.298039, 0.588235, 0.486275, 0.321569, 0.054902, 0.52549, 0.0745098]]</td>\n",
" <td>[[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]]</td>\n",
" <td>4</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.607843, 0.309804, 0.956863, 0.0705882, 0.0901961, 0.27451, 0.545098, 0.576471, 0.741176, 0.827451, 0.988235, 0.25098], [0.603922, 0.207843, 0.254902, 0.635294, 0.160784, 0.592157, 0.396078, 0.0156863, 0.196078, 0.619608, 0.752941, 0.843137], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.501961, 0.278431, 0.635294, 0.215686, 0.45098, 0.0313726, 0.780392, 0.835294, 0.721569, 0.435294, 0.172549, 0.239216]]</td>\n",
" <td>[[1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0]]</td>\n",
" <td>5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.286275], [0.835294, 0.690196, 0.639216, 0.227451, 0.372549, 0.294118, 0.0509804, 0.203922, 0.756863, 0.815686, 0.956863, 0.564706], [0.635294, 0.337255, 0.419608, 0.607843, 0.780392, 0.639216, 0.541176, 0.00392157, 0.784314, 0.984314, 0.509804, 0.776471], [0.843137, 0.85098, 0.972549, 0.92549, 0.227451, 0.980392, 0.823529, 0.388235, 0.631373, 0.00784314, 0.701961, 0.14902], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686]], [[0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0], [1, 0, 0], [0, 1, 0]], 0),\n",
" ([[0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.286275, 0.388235, 0.0352941, 0.0745098, 0.0862745, 0.545098, 0.890196, 0.360784, 0.309804, 0.733333, 0.984314, 0.317647], [0.960784, 0.839216, 0.0588235, 0.870588, 0.101961, 0.666667, 0.176471, 0.054902, 0.737255, 0.266667, 0.329412, 0.278431], [0.933333, 0.0470588, 0.964706, 0.752941, 0.443137, 0.564706, 0.960784, 0.701961, 0.196078, 0.113725, 0.286275, 0.596078], [0.584314, 0.760784, 0.227451, 0.0313726, 0.32549, 0.694118, 0.639216, 0.294118, 0.929412, 0.498039, 0.027451, 0.505882], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098]], [[0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]], 1),\n",
" ([[0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.0666667, 0.101961, 0.827451, 0.313726, 0.168627, 0.972549, 0.521569, 0.0431373, 0.227451, 0.376471, 0.929412, 0.717647], [0.764706, 0.866667, 0.486275, 0.913726, 0.517647, 0.113725, 0.247059, 0.937255, 0.72549, 0.0235294, 0.572549, 0.258824], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.278431, 0.854902, 0.623529, 0.184314, 0.270588, 0.45098, 0.870588, 0.909804, 0.682353, 0.239216, 0.2, 0.733333]], [[0, 0, 1], [1, 0, 0], [0, 1, 0], [1, 0, 0], [1, 0, 0], [0, 1, 0], [0, 1, 0], [0, 1, 0], [0, 0, 1]], 2),\n",
" ([[0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [0.317647, 0.470588, 0.486275, 0.466667, 0.282353, 0.411765, 0.513726, 0.247059, 0.160784, 0.956863, 0.811765, 0.113725], [0.34902, 0.239216, 0.537255, 0.12549, 0.282353, 0.729412, 0.164706, 0.839216, 0.478431, 0.376471, 0.588235, 0.0156863], [0.768628, 0.345098, 0.00392157, 0.380392, 0.592157, 0.290196, 0.768628, 0.627451, 0.368627, 0.854902, 0.168627, 0.254902], [0.333333, 0.764706, 0.862745, 0.611765, 0.223529, 0.737255, 0.647059, 0.917647, 0.0901961, 0.00784314, 0.439216, 0.0823529], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314]], [[1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0]], 3),\n",
" ([[0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0.454902], [0.00392157, 0.231373, 0.941177, 0.6, 0.364706, 0.419608, 0.811765, 0.243137, 0.745098, 0.552941, 0.968628, 0.913726], [0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.219608, 0.933333, 0.223529, 0.145098, 0.443137, 0.505882, 1.0, 0.0627451, 0.690196, 0.266667, 0.513726, 0.556863], [0.254902, 0.792157, 0.87451, 0.396078, 0.192157, 0.635294, 0.254902, 0.67451, 0.545098, 0.772549, 0.788235, 0.792157], [0.407843, 0.607843, 0.152941, 0.913726, 0.972549, 0.298039, 0.588235, 0.486275, 0.321569, 0.054902, 0.52549, 0.0745098]], [[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]], 4),\n",
" ([[0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.607843, 0.309804, 0.956863, 0.0705882, 0.0901961, 0.27451, 0.545098, 0.576471, 0.741176, 0.827451, 0.988235, 0.25098], [0.603922, 0.207843, 0.254902, 0.635294, 0.160784, 0.592157, 0.396078, 0.0156863, 0.196078, 0.619608, 0.752941, 0.843137], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.501961, 0.278431, 0.635294, 0.215686, 0.45098, 0.0313726, 0.780392, 0.835294, 0.721569, 0.435294, 0.172549, 0.239216]], [[1, 0, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0], [0, 1, 0], [0, 0, 1], [0, 0, 1], [0, 1, 0]], 5)]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
"\n",
"SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
" 'image_data_packed', -- Output table\n",
" 'species', -- Dependent variable\n",
" 'rgb', -- Independent variable\n",
" 10, -- Buffer size\n",
" 255 -- Normalizing constant\n",
" );\n",
"\n",
"SELECT * FROM image_data_packed ORDER BY buffer_id;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Review the output summary data:"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>source_table</th>\n",
" <th>output_table</th>\n",
" <th>dependent_varname</th>\n",
" <th>independent_varname</th>\n",
" <th>dependent_vartype</th>\n",
" <th>class_values</th>\n",
" <th>buffer_size</th>\n",
" <th>normalizing_const</th>\n",
" <th>num_classes</th>\n",
" </tr>\n",
" <tr>\n",
" <td>image_data</td>\n",
" <td>image_data_packed</td>\n",
" <td>species</td>\n",
" <td>rgb</td>\n",
" <td>text</td>\n",
" <td>[u'bird', u'cat', u'dog']</td>\n",
" <td>10</td>\n",
" <td>255.0</td>\n",
" <td>3</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog'], 10, Decimal('255.0'), 3)]"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"SELECT * FROM image_data_packed_summary;"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"<a id=\"set_num_classes\"></a>\n",
"# 8. Setting number of classes\n",
"\n",
"If want the 1-hot encoded vector to have more classes than present in the data, use the num_classes param "
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"1 rows affected.\n",
"2 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>independent_var</th>\n",
" <th>dependent_var</th>\n",
" <th>buffer_id</th>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [0.835294, 0.690196, 0.639216, 0.227451, 0.372549, 0.294118, 0.0509804, 0.203922, 0.756863, 0.815686, 0.956863, 0.564706], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.764706, 0.866667, 0.486275, 0.913726, 0.517647, 0.113725, 0.247059, 0.937255, 0.72549, 0.0235294, 0.572549, 0.258824], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0.454902], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.843137, 0.85098, 0.972549, 0.92549, 0.227451, 0.980392, 0.823529, 0.388235, 0.631373, 0.00784314, 0.701961, 0.14902], [0.407843, 0.607843, 0.152941, 0.913726, 0.972549, 0.298039, 0.588235, 0.486275, 0.321569, 0.054902, 0.52549, 0.0745098], [0.0666667, 0.101961, 0.827451, 0.313726, 0.168627, 0.972549, 0.521569, 0.0431373, 0.227451, 0.376471, 0.929412, 0.717647], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.635294, 0.337255, 0.419608, 0.607843, 0.780392, 0.639216, 0.541176, 0.00392157, 0.784314, 0.984314, 0.509804, 0.776471], [0.00392157, 0.231373, 0.941177, 0.6, 0.364706, 0.419608, 0.811765, 0.243137, 0.745098, 0.552941, 0.968628, 0.913726], [0.278431, 0.854902, 0.623529, 0.184314, 0.270588, 0.45098, 0.870588, 0.909804, 0.682353, 0.239216, 0.2, 0.733333], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.219608, 0.933333, 0.223529, 0.145098, 0.443137, 0.505882, 1.0, 0.0627451, 0.690196, 0.266667, 0.513726, 0.556863], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.286275], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.254902, 0.792157, 0.87451, 0.396078, 0.192157, 0.635294, 0.254902, 0.67451, 0.545098, 0.772549, 0.788235, 0.792157]]</td>\n",
" <td>[[0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>[[0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.933333, 0.0470588, 0.964706, 0.752941, 0.443137, 0.564706, 0.960784, 0.701961, 0.196078, 0.113725, 0.286275, 0.596078], [0.768628, 0.345098, 0.00392157, 0.380392, 0.592157, 0.290196, 0.768628, 0.627451, 0.368627, 0.854902, 0.168627, 0.254902], [0.584314, 0.760784, 0.227451, 0.0313726, 0.32549, 0.694118, 0.639216, 0.294118, 0.929412, 0.498039, 0.027451, 0.505882], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.603922, 0.207843, 0.254902, 0.635294, 0.160784, 0.592157, 0.396078, 0.0156863, 0.196078, 0.619608, 0.752941, 0.843137], [0.501961, 0.278431, 0.635294, 0.215686, 0.45098, 0.0313726, 0.780392, 0.835294, 0.721569, 0.435294, 0.172549, 0.239216], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098], [0.286275, 0.388235, 0.0352941, 0.0745098, 0.0862745, 0.545098, 0.890196, 0.360784, 0.309804, 0.733333, 0.984314, 0.317647], [0.960784, 0.839216, 0.0588235, 0.870588, 0.101961, 0.666667, 0.176471, 0.054902, 0.737255, 0.266667, 0.329412, 0.278431], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.34902, 0.239216, 0.537255, 0.12549, 0.282353, 0.729412, 0.164706, 0.839216, 0.478431, 0.376471, 0.588235, 0.0156863], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.607843, 0.309804, 0.956863, 0.0705882, 0.0901961, 0.27451, 0.545098, 0.576471, 0.741176, 0.827451, 0.988235, 0.25098], [0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.317647, 0.470588, 0.486275, 0.466667, 0.282353, 0.411765, 0.513726, 0.247059, 0.160784, 0.956863, 0.811765, 0.113725], [0.333333, 0.764706, 0.862745, 0.611765, 0.223529, 0.737255, 0.647059, 0.917647, 0.0901961, 0.00784314, 0.439216, 0.0823529]]</td>\n",
" <td>[[0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0]]</td>\n",
" <td>1</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[([[0.258824, 0.478431, 0.772549, 0.105882, 0.152941, 0.345098, 0.803922, 0.729412, 0.972549, 0.764706, 0.235294, 0.482353], [0.482353, 0.211765, 0.886275, 0.32549, 0.745098, 0.72549, 0.172549, 0.717647, 0.647059, 0.4, 0.694118, 0.466667], [0.72549, 0.603922, 0.207843, 0.631373, 0.733333, 0.792157, 0.913726, 0.443137, 0.384314, 0.14902, 0.407843, 0.772549], [0.580392, 0.282353, 0.713726, 0.596078, 0.239216, 0.968628, 0.388235, 0.109804, 0.360784, 0.576471, 0.745098, 0.615686], [0.152941, 0.870588, 0.623529, 0.917647, 0.384314, 0.345098, 0.596078, 0.494118, 0.45098, 0.388235, 0.862745, 0.0313726], [0.835294, 0.690196, 0.639216, 0.227451, 0.372549, 0.294118, 0.0509804, 0.203922, 0.756863, 0.815686, 0.956863, 0.564706], [0.72549, 0.682353, 0.109804, 0.105882, 0.796079, 0.368627, 0.584314, 0.564706, 0.47451, 0.733333, 0.909804, 0.27451], [0.764706, 0.866667, 0.486275, 0.913726, 0.517647, 0.113725, 0.247059, 0.937255, 0.72549, 0.0235294, 0.572549, 0.258824], [0.478431, 0.988235, 0.188235, 0.796079, 0.0901961, 0.913726, 0.4, 0.298039, 0.545098, 0.12549, 0.0823529, 0.454902], [0.145098, 0.678431, 0.517647, 0.0235294, 0.470588, 0.0392157, 0.756863, 0.435294, 0.815686, 0.698039, 0.882353, 0.572549], [0.380392, 0.866667, 0.529412, 0.760784, 0.541176, 0.647059, 0.407843, 0.54902, 0.0352941, 0.894118, 0.619608, 0.533333], [0.843137, 0.85098, 0.972549, 0.92549, 0.227451, 0.980392, 0.823529, 0.388235, 0.631373, 0.00784314, 0.701961, 0.14902], [0.407843, 0.607843, 0.152941, 0.913726, 0.972549, 0.298039, 0.588235, 0.486275, 0.321569, 0.054902, 0.52549, 0.0745098], [0.0666667, 0.101961, 0.827451, 0.313726, 0.168627, 0.972549, 0.521569, 0.0431373, 0.227451, 0.376471, 0.929412, 0.717647], [0.576471, 0.701961, 0.580392, 0.627451, 0.964706, 0.509804, 0.517647, 0.564706, 0.6, 0.152941, 0.690196, 0.215686], [0.635294, 0.337255, 0.419608, 0.607843, 0.780392, 0.639216, 0.541176, 0.00392157, 0.784314, 0.984314, 0.509804, 0.776471], [0.00392157, 0.231373, 0.941177, 0.6, 0.364706, 0.419608, 0.811765, 0.243137, 0.745098, 0.552941, 0.968628, 0.913726], [0.278431, 0.854902, 0.623529, 0.184314, 0.270588, 0.45098, 0.870588, 0.909804, 0.682353, 0.239216, 0.2, 0.733333], [0.00392157, 0.901961, 0.160784, 0.654902, 0.184314, 0.313726, 0.521569, 0.807843, 0.227451, 0.905882, 0.152941, 0.823529], [0.658824, 0.827451, 0.835294, 0.462745, 0.764706, 0.752941, 0.811765, 0.901961, 0.113725, 0.215686, 0.964706, 0.0235294], [0.282353, 0.407843, 0.635294, 0.52549, 0.556863, 0.0117647, 0.384314, 0.0862745, 0.772549, 0.92549, 0.729412, 0.176471], [0.219608, 0.933333, 0.223529, 0.145098, 0.443137, 0.505882, 1.0, 0.0627451, 0.690196, 0.266667, 0.513726, 0.556863], [0.521569, 0.733333, 0.968628, 0.776471, 0.945098, 0.443137, 0.760784, 0.129412, 0.235294, 0.847059, 0.0392157, 0.635294], [0.145098, 0.203922, 0.878431, 0.258824, 0.858824, 0.882353, 0.490196, 0.796079, 0.478431, 0.854902, 0.215686, 0.286275], [0.113725, 0.647059, 0.00392157, 0.396078, 0.486275, 0.0705882, 0.494118, 0.309804, 0.384314, 0.666667, 0.278431, 0.905882], [0.254902, 0.792157, 0.87451, 0.396078, 0.192157, 0.635294, 0.254902, 0.67451, 0.545098, 0.772549, 0.788235, 0.792157]], [[0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]], 0),\n",
" ([[0.509804, 0.815686, 0.270588, 0.768628, 0.843137, 0.623529, 0.00784314, 0.376471, 0.74902, 0.290196, 0.101961, 0.909804], [0.831373, 0.635294, 0.0117647, 0.0470588, 0.831373, 0.411765, 0.0784314, 0.466667, 0.372549, 0.223529, 0.337255, 0.764706], [0.192157, 0.0392157, 0.556863, 0.74902, 0.211765, 0.74902, 0.541176, 0.588235, 0.67451, 0.776471, 0.917647, 0.137255], [0.164706, 0.337255, 0.360784, 0.619608, 0.529412, 0.533333, 0.470588, 0.556863, 0.498039, 0.929412, 0.109804, 0.905882], [0.933333, 0.0470588, 0.964706, 0.752941, 0.443137, 0.564706, 0.960784, 0.701961, 0.196078, 0.113725, 0.286275, 0.596078], [0.768628, 0.345098, 0.00392157, 0.380392, 0.592157, 0.290196, 0.768628, 0.627451, 0.368627, 0.854902, 0.168627, 0.254902], [0.584314, 0.760784, 0.227451, 0.0313726, 0.32549, 0.694118, 0.639216, 0.294118, 0.929412, 0.498039, 0.027451, 0.505882], [0.403922, 0.564706, 0.384314, 0.690196, 0.658824, 0.341176, 0.521569, 0.717647, 0.207843, 0.623529, 0.380392, 0.380392], [0.4, 0.0352941, 0.639216, 0.968628, 0.141176, 0.87451, 0.388235, 0.294118, 0.388235, 0.921569, 0.6, 0.384314], [0.603922, 0.207843, 0.254902, 0.635294, 0.160784, 0.592157, 0.396078, 0.0156863, 0.196078, 0.619608, 0.752941, 0.843137], [0.501961, 0.278431, 0.635294, 0.215686, 0.45098, 0.0313726, 0.780392, 0.835294, 0.721569, 0.435294, 0.172549, 0.239216], [0.878431, 0.0392157, 0.694118, 0.356863, 0.364706, 0.52549, 0.996078, 0.372549, 0.568627, 0.823529, 0.784314, 0.65098], [0.286275, 0.388235, 0.0352941, 0.0745098, 0.0862745, 0.545098, 0.890196, 0.360784, 0.309804, 0.733333, 0.984314, 0.317647], [0.960784, 0.839216, 0.0588235, 0.870588, 0.101961, 0.666667, 0.176471, 0.054902, 0.737255, 0.266667, 0.329412, 0.278431], [0.113725, 0.419608, 0.862745, 0.74902, 0.560784, 0.443137, 0.509804, 0.788235, 0.478431, 0.831373, 0.478431, 0.109804], [0.0666667, 0.780392, 0.67451, 0.0901961, 0.894118, 0.839216, 0.431373, 0.254902, 0.454902, 0.960784, 0.784314, 0.929412], [0.960784, 0.545098, 0.396078, 0.521569, 0.254902, 0.458824, 0.298039, 0.933333, 0.54902, 0.192157, 0.768628, 0.980392], [0.231373, 0.423529, 0.152941, 0.521569, 0.533333, 0.305882, 0.996078, 0.2, 0.662745, 0.0941177, 0.654902, 0.85098], [0.0, 0.192157, 0.478431, 0.905882, 0.901961, 0.980392, 0.180392, 0.533333, 0.192157, 0.631373, 0.564706, 0.976471], [0.168627, 0.858824, 0.027451, 0.972549, 0.458824, 0.556863, 0.407843, 0.494118, 0.721569, 0.784314, 0.219608, 0.4], [0.34902, 0.239216, 0.537255, 0.12549, 0.282353, 0.729412, 0.164706, 0.839216, 0.478431, 0.376471, 0.588235, 0.0156863], [0.192157, 0.796079, 0.376471, 0.92549, 0.235294, 0.329412, 0.470588, 0.627451, 0.85098, 0.72549, 0.0823529, 0.14902], [0.607843, 0.309804, 0.956863, 0.0705882, 0.0901961, 0.27451, 0.545098, 0.576471, 0.741176, 0.827451, 0.988235, 0.25098], [0.466667, 0.603922, 0.972549, 0.235294, 0.866667, 0.737255, 0.580392, 0.870588, 0.113725, 0.168627, 0.156863, 0.882353], [0.317647, 0.470588, 0.486275, 0.466667, 0.282353, 0.411765, 0.513726, 0.247059, 0.160784, 0.956863, 0.811765, 0.113725], [0.333333, 0.764706, 0.862745, 0.611765, 0.223529, 0.737255, 0.647059, 0.917647, 0.0901961, 0.00784314, 0.439216, 0.0823529]], [[0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0]], 1)]"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;\n",
"\n",
"SELECT madlib.training_preprocessor_dl('image_data', -- Source table\n",
" 'image_data_packed', -- Output table\n",
" 'species', -- Dependent variable\n",
" 'rgb', -- Independent variable\n",
" NULL, -- Buffer size\n",
" 255, -- Normalizing constant\n",
" 5 -- Number of desired class values\n",
" );\n",
"\n",
"SELECT * FROM image_data_packed ORDER BY buffer_id;"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1 rows affected.\n"
]
},
{
"data": {
"text/html": [
"<table>\n",
" <tr>\n",
" <th>source_table</th>\n",
" <th>output_table</th>\n",
" <th>dependent_varname</th>\n",
" <th>independent_varname</th>\n",
" <th>dependent_vartype</th>\n",
" <th>class_values</th>\n",
" <th>buffer_size</th>\n",
" <th>normalizing_const</th>\n",
" <th>num_classes</th>\n",
" </tr>\n",
" <tr>\n",
" <td>image_data</td>\n",
" <td>image_data_packed</td>\n",
" <td>species</td>\n",
" <td>rgb</td>\n",
" <td>text</td>\n",
" <td>[u'bird', u'cat', u'dog', None, None]</td>\n",
" <td>26</td>\n",
" <td>255.0</td>\n",
" <td>5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
"[(u'image_data', u'image_data_packed', u'species', u'rgb', u'text', [u'bird', u'cat', u'dog', None, None], 26, Decimal('255.0'), 5)]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"SELECT * FROM image_data_packed_summary;"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 1
}