scripts/datagen/genRandData4DescriptiveStats.dml

#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

/*
------------------------------------------------
  Parameters                                      
------------------------------------------------
$R          = #rows
$C          = #columns
$NC         = number of categorical attributes
$MAXDOMAIN  = maximum domain size
$DATA       = output file path on HDFS
$SETSIZE    = Size of one bivariate set
$LABELSETSIZE= Size of second bivariate set with labels
$TYPES      = output attribute types
$TYPES1     = Attribute types for Set1
$TYPES2     = Attribute types for Set2
$INDEX1     = Indices for Set1
$INDEX2     = Indices for Set2
$FMT        = output format
------------------------------------------------
hadoop jar SystemML.jar -f genData4Stats.dml -nvargs R=1000000 C=1000 NC=50 MAXDOMAIN=1100 DATA=stats/data TYPES=stats/types SETSIZE=15 LABELSETSIZE=10 TYPES1=... Types2=... INDEX1=.. INDEX2=..FMT=csv
------------------------------------------------
*/


FMT = ifdef($FMT,"binary"); # default output format

# number of categorical attributes.. numC <= C
numC = $NC;
numO = as.integer(numC/2);
numNominal = numC - numO;
print("Categorical Mix = (" + numC + "," + numO + "," + numNominal +")");

# maximum domain size among all categorical attributes
maxDomainSize = $MAXDOMAIN;

# Divide $C attributes according to the following logic:
#
#   1     numO       numC               C
#   |-------|---------|-----------------|
#      ord    nominal    scale
#
# numC+1-$C: scale
# 1-numC/2: ordinal
# (numC/2+1)-numC: nominal

types = matrix(1, rows=1, cols=$C);
ocutoff = numO;
types[1,1:ocutoff] = matrix(1,rows=1,cols=ocutoff)*3;
types[1, ocutoff+1:numC] = matrix(1,rows=1,cols=(numC-ocutoff))*2;

# Generate data
A = rand(rows=$R, cols=$C, sparsity=1);
B = matrix(0,rows=nrow(A), cols=ncol(A));
parfor (i in 1:numC) {
    Ai = A[,i];

    tmp = round(rand(rows=1,cols=1, min=1, max=maxDomainSize));
    domain = as.scalar(tmp[1,1]);

    # for some attributes, choose the maxDomainSize
    tmp = rand(rows=1,cols=1);
    if (as.scalar(tmp[1,1]) < 0.5) {
        domain = maxDomainSize;
    }

    B[,i] = round(1+(domain-1)*Ai);
}
B[ ,(numC+1):ncol(A)] = A[, (numC+1):ncol(A)];
 

write(B, $DATA, format=FMT);
write(types, $TYPES, format=FMT);

# ----- Generator for Bivariate ---------

settypes1 = matrix(1, rows=1, cols=$SETSIZE);
index1   = matrix(0, rows=1, cols=$SETSIZE);

catSetSize = as.integer($SETSIZE/2);
ocutoff = as.integer(catSetSize/2);
print("Set Mix = (" + $SETSIZE + "," + catSetSize + "," + ocutoff + ")" );
settypes1[1, 1:ocutoff] = matrix(1,rows=1,cols=ocutoff)*3;
settypes1[1, ocutoff+1:catSetSize] = matrix(1,rows=1,cols=(catSetSize-ocutoff))*2;

# select ordinal indices
tmp = rand(rows=1, cols=ocutoff);
index1[1, 1:ocutoff] = round(1 + (numO-1)*tmp);

# select nominal indices
nominalSetSize = catSetSize-ocutoff;
tmp = rand(rows=1, cols=nominalSetSize);
index1[1, ocutoff+1:catSetSize] = round(numO+1 + (numC-numO-1)*tmp);

# select scale attributes
scaleSetSize = $SETSIZE-catSetSize;
tmp = rand(rows=1, cols=scaleSetSize);
index1[1, catSetSize+1:$SETSIZE] = round(numC+1 + ($C-numC-1)*tmp);


# --- select types and indices for LABELSET
settypes2 = matrix(2, rows=1, cols=$LABELSETSIZE);
index2   = matrix(0, rows=1, cols=$LABELSETSIZE);
if($LABELSETSIZE > 1) {
    settypes2[1,1] = 1;
    r = as.scalar(rand(rows=1,cols=1));
    index2[1,1] = round(numC+1 + ($C-numC-1)*r)
}
else {
    r = as.scalar(rand(rows=1,cols=1));
    index2[1,1] = round( numO+1 + (numC-numO-1)*r )
}

for(i in 2:as.integer($LABELSETSIZE/2)) {
    settypes2[1,i] = 3;
    r = as.scalar(rand(rows=1,cols=1));
    index2[1,i] = round( 1 + (numO-1)*r )
}

for(i in as.integer($LABELSETSIZE/2)+1:$LABELSETSIZE) {
    settypes2[1,i] = 2;
    r = as.scalar(rand(rows=1,cols=1));
    index2[1,i] = round( numO+1 + (numC-numO-1)*r )
}

write(settypes1, $TYPES1, format=FMT);
write(settypes2, $TYPES2, format=FMT);
write(index1, $INDEX1, format=FMT);
write(index2, $INDEX2, format=FMT);