asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/dblp-3_1.2.aql

drop dataverse fuzzyjoin if exists;

create dataverse fuzzyjoin;

use dataverse fuzzyjoin;

create type DBLPType as open {
  id: int32, 
  dblpid: string,
  title: string,
  authors: string,
  misc: string
}

create dataset DBLP(DBLPType) partitioned by key id;

load dataset DBLP 
using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
(("path"="nc1://data/pub-small/dblp-small-id.txt"),("format"="delimited-text"),("delimiter"=":"));

write output to nc1:'rttest/fuzzyjoin_dblp-3_1.2.adm';

//
// -- - Stage 3 - --
//
for $ridpair in 
    //
    // -- - Stage 2 - --
    //
    for $paperLeft in dataset('DBLP')
    let $lenLeft := len(counthashed-word-tokens($paperLeft.title))
    let $tokensLeft :=
        for $tokenUnranked in counthashed-word-tokens($paperLeft.title)
        for $tokenRanked at $i in
            //
            // -- - Stage 1 - --
            //
            for $paper in dataset('DBLP')
            for $token in counthashed-word-tokens($paper.title)
            /*+ hash */
            group by $tokenGroupped := $token with $paper
            order by count($paper), $tokenGroupped
            return $tokenGroupped
        where $tokenUnranked = /*+ bcast */ $tokenRanked
        order by $i
        return $i
    for $prefixTokenLeft in subset-collection(
                                $tokensLeft, 
                                0,
                                prefix-len-jaccard($lenLeft, .5f))

    for $paperRight in dataset('DBLP')
    let $lenRight := len(counthashed-word-tokens($paperRight.title))
    let $tokensRight :=
        for $tokenUnranked in counthashed-word-tokens($paperRight.title)
        for $tokenRanked at $i in
            //
            // -- - Stage 1 - --
            //
            for $paper in dataset('DBLP')
            for $token in counthashed-word-tokens($paper.title)
            /*+ hash */
            group by $tokenGroupped := $token with $paper
            order by count($paper), $tokenGroupped
            return $tokenGroupped
        where $tokenUnranked = /*+ bcast */ $tokenRanked
        order by $i
        return $i
    for $prefixTokenRight in subset-collection(
                                $tokensRight, 
                                0,
                                prefix-len-jaccard($lenRight, .5f))

    where $prefixTokenLeft = $prefixTokenRight

    let $sim := similarity-jaccard-prefix(
                    $lenLeft,
                    $tokensLeft,
                    $lenRight,
                    $tokensRight,
                    $prefixTokenLeft,
                    .5f)
    where $sim >= .5f and $paperLeft.id < $paperRight.id
    /*+ hash */
    group by $idLeft := $paperLeft.id, $idRight := $paperRight.id with $sim
    return {'idLeft': $idLeft, 'idRight': $idRight, 'sim': $sim[0]}

for $paperLeft in dataset('DBLP')
for $paperRight in dataset('DBLP')
where $ridpair.idLeft = $paperLeft.id and $ridpair.idRight = $paperRight.id
order by $paperLeft.id, $paperRight.id
return {'left': $paperLeft, 'right': $paperRight, 'sim': $ridpair.sim}